Optimization for image thumbnails

.gitignore

@@ -1,3 +1,4 @@
 .next/
 node_modules/
 .env
+cache/

api/images.js

@@ -1,14 +1,63 @@
 const router = require('express-promise-router')()
 const db = require('../db')
 const ensureAdmin = require('./middleware/ensureAdmin')
+const fs = require('fs');
+const path = require('path');
+
+const cacheRoot = path.join(__dirname, '../cache/images')
+
+fs.promises.mkdir(cacheRoot, {recursive: true})
+  .catch(() => console.error("Could not create image cache dir: " + cacheRoot));
+
+const memCache = {};
+
+async function writeImageFile(uuid, size, data) {
+  if(size !== 'thumb' && size !== 'large')
+    throw new Error(`Unknown image size ${size}`);
+
+  await fs.promises.writeFile(path.join(cacheRoot, `${uuid}-${size}`), data);
+}
+
+async function getImage(uuid, size) {
+  // If we've already stored a mime type, that means we've written the file to disk
+  if(memCache[`${uuid}-${size}`]?.mime_type) {
+    return {
+      file: fs.createReadStream(path.join(cacheRoot, `${uuid}-${size}`)),
+      mime_type: memCache[`${uuid}-${size}`].mime_type
+    }
+  }
+
+  // If we've stored a promise, that means we're currently retrieving it from the db
+  if(memCache[`${uuid}-${size}`]?.dbPromise) {
+    return await memCache[`${uuid}-${size}`]?.dbPromise;
+  }
+
+  // Otherwise, retrieve it from the DB, and then write it to disk
+  const record = memCache[`${uuid}-${size}`] = {};
+  record.dbPromise = db.item.getImage(uuid, size);
+
+  const image = await record.dbPromise;
+  await writeImageFile(uuid, size, image.file);
+
+  record.mime_type = image.mime_type;
+  delete record.dbPromise;
+
+  return image;
+}
 
 router.get('/:uuid/:size', async (req, res) => {
-  const image = await db.item.getImage(req.params.uuid, req.params.size)
+  const image = await getImage(req.params.uuid, req.params.size)
 
   const cacheSeconds = 60 * 60 * 24;
   res.set('Cache-Control', cacheSeconds);
   res.set('Content-Type', image.mime_type)
-  res.end(image.file)
+
+  if(Buffer.isBuffer(image.file))
+    res.end(image.file)
+  else if (typeof image.file.pipe === 'function')
+    image.file.pipe(res)
+  else
+    throw new Error("Unable to send file to user");
 })
 
 router.post('/:uuid/featured', ensureAdmin, async (req, res) => {

components/card/style.module.css

@@ -74,6 +74,8 @@
 
 .card img{
   width:80%;
+  aspect-ratio: 4 / 3;
+  object-fit: cover;
   padding:10px;
   background: #d4d4fb;
   border: solid 1px #dec8e2;

import/.gitignore

@@ -1,3 +0,0 @@
-users.json
-node_modules/
-datafile.xlsx

import/index.js

@@ -1,45 +0,0 @@
-#!/usr/bin/env node
-const path = require('path')
-require('dotenv').config({path: path.join(__dirname, '../.env')})
-
-const pg = require('../db/pg')
-const newDB = require('../db')
-const oldDB = require('./mongo')('mongodb://localhost/sos')
-
-const createItems = require('./tasks/createItems')
-const createUsers = require('./tasks/createUsers')
-const uploadImages = require('./tasks/uploadImages')
-const saveExcelDataFile = require('./tasks/writeExcelSheet')
-
-async function doMigration() {
-  const items = await oldDB.sock.find().lean().exec()
-  const allUsers = await oldDB.user.find().populate('purchases').lean().exec()
-  const carts = allUsers.map(u => u.purchases).flat()
-  const coupons = await oldDB.coupon.find().lean().exec()
-
-  const registeredUsers = allUsers.filter(user => user.email)
-
-  console.log(`Loaded ${carts.length} purchases and ${registeredUsers.length} users`)
-
-  console.log(`Inserting ${registeredUsers.length} users into the database`)
-  const importAdmin = await createUsers(newDB, registeredUsers);
-  console.log(`  Found user account ${importAdmin.uuid} (${importAdmin.email}) to attribute file uploads to`)
-
-  console.log(`\nInserting ${items.length} items into database`)
-  const itemImages = await createItems(newDB, items);
-
-  const numImages = itemImages.map(({images}) => images.length).reduce((a,b) => b+a, 0)
-  console.log(`\nImporting ${numImages} images into database`)
-  await uploadImages(newDB, itemImages, importAdmin.uuid)
-
-  console.log('\nWriting Excel data file')
-  await saveExcelDataFile(allUsers, coupons, items, path.join(__dirname, './datafile.xlsx'))
-}
-
-doMigration()
-  .catch(console.error)
-  .finally(() => {
-    pg.end()
-    oldDB._connection.close()
-  })
-

import/mongo/cart.js

@@ -1,24 +0,0 @@
-var mongoose = require('mongoose');
-var Schema = mongoose.Schema;
-
-module.exports.schema = new Schema({
-	user: {type: Schema.Types.ObjectId, ref: 'User'},
-  items: [{type: Schema.Types.ObjectId, required: true, ref: 'Sock'}],
-	purchased: {type: Schema.Types.Mixed, enum: [true, false, 'refunded'], default: false, required: true},
-	purchaseTime: {type: Number},
-	shipped: {type: Boolean, required: true, default: false},
-	shippedOn: {type: Number},
-	address: {type: String}, // Easypost id
-	shipment: {type: String}, // Easypost id
-	shipmentMeasured: {type: Boolean, default: false},
-	sockPrice: {type: Number},
-	totalPrice: {type: Number},
-	shippingEstimate: {type: Number},
-	coupon: {type: Schema.Types.ObjectId, required: false, ref: 'Coupon'},
-	trackingCode: {type: String},
-	needsCustoms: {type: Boolean, default: false}
-}, {
-	usePushEach: true
-});
-
-module.exports.model = mongoose.model('Cart', module.exports.schema);

import/mongo/coupon.js

@@ -1,20 +0,0 @@
-var mongoose = require('mongoose');
-var Schema = mongoose.Schema;
-
-module.exports.schema = new Schema({
-	code: {type: String, required: true, unique: true, index: true},
-
-	numAllowedUses: {type: Number, required: true, default: 1},
-	uses: [{type: Schema.Types.ObjectId, required: true, ref: 'Cart'}],
-	expires: {type: Number, required: true, default: 0},
-
-	flatDiscount: {type: Number, required: true, default: 0},
-	percentDiscount: {type: Number, required: true, default:0},
-	socksFree: {type: Number, required: true, default: 0},
-	perSockDiscount: {type: Number, required: false, default: 0},
-	freeShipping: {type: Boolean, required: true, default: false}
-}, {
-	usePushEach: true
-});
-
-module.exports.model = mongoose.model('Coupon', module.exports.schema);

import/mongo/index.js

@@ -1,31 +0,0 @@
-const mongoose = require('mongoose')
-mongoose.Promise = Promise
-
-const models = {
-  Sock: require('./sock.js').schema,
-  Media: require('./media.js').schema,
-  User: require('./user.js').schema,
-  Cart: require('./cart.js').schema,
-  Coupon: require('./coupon.js').schema,
-}
-
-module.exports = function connect(url) {
-  const connection = mongoose.createConnection(url)
-
-  connection.on('error', console.error.bind(console, 'connection error:'));
-
-  var sock = connection.model('Sock',models.Sock, 'socks');
-  var media = connection.model('Media',models.Media, 'medias');
-  var user = connection.model('User',models.User, 'users');
-  var cart = connection.model('Cart',models.Cart, 'carts');
-  var coupon = connection.model('Coupon',models.Coupon, 'coupons');
-
-  return {
-    sock,
-    media,
-    user,
-    cart,
-    coupon,
-    _connection: connection
-  }
-}

import/mongo/media.js

@@ -1,18 +0,0 @@
-var mongoose = require('mongoose');
-var Schema = mongoose.Schema;
-
-var MediaSchema = new Schema({
-	name: {type: String, required: true},
-	urlslug: {type: String, required: true, unique: true},
-	uploaded: {type: Number, required: true},
-	filename: {type: String, required: true, unique: true},
-	thumbnail: {type: String},
-	mimetype: {type: String, required: true},
-	width: {type:Number,required:true},
-	height: {type:Number,required:true}
-});
-
-var Media = mongoose.model('Media', MediaSchema);
-
-module.exports.model = Media;
-module.exports.schema = MediaSchema

import/mongo/sock.js

@@ -1,20 +0,0 @@
-var mongoose = require('mongoose');
-var Schema = mongoose.Schema;
-
-module.exports.schema = new Schema({
-	name: {type: String, required: true},
-	urlslug: {type: String, required: true, lowercase: true, unique: true},
-	productImage: {type: String, required: true},
-	description: {type: String, required: true},
-	numberInStock: {type: Number},
-	price: {type: Number},
-	images: [String], //Does not include productImage
-	tags: [String],
-	publishTime:{type: Number, required: true},
-	expireTime:{type: Number, required: true},
-	preorderDate: {type: Number, default: null}
-});
-
-module.exports.schema.index({name: 'text', description: 'text'});
-
-module.exports.model = mongoose.model('Sock', module.exports.schema);

import/mongo/user.js

@@ -1,19 +0,0 @@
-var mongoose = require('mongoose');
-var Schema = mongoose.Schema;
-
-module.exports.schema = new Schema({
-	email: {type: String},
-	emailConfirmed: {type: Boolean, default: false},
-	emailPin: {type: String},
-  password: {type: String},
-  isAdmin: {type: Boolean, default: false},
-	cart: {type: Schema.Types.ObjectId, ref: 'Cart'},
-  purchases: [{type: Schema.Types.ObjectId, ref: 'Cart'}],
-  credit: {type: Number, default: 0},//Credit in cents because non-integers are confusing?
-	savedAddress: {type: String},
-	lastLogin: {type: Number, required: true}
-}, {
-	usePushEach: true
-});
-
-module.exports.model = mongoose.model('User', module.exports.schema);

import/package.json

@@ -1,8 +0,0 @@
-{
-  "name": "sos-import",
-  "version": "0.0.1",
-  "dependencies": {
-    "exceljs": "^4.2.1",
-    "mongoose": "^4.4.16"
-  }
-}

import/tariffData.json

@@ -1,8 +0,0 @@
-{
-  "sock": {
-    "hs_tariff_number": "611595",
-    "customs_description": "A pair of socks",
-    "origin_country": "CN",
-    "weight": 2
-  }
-}

import/tasks/createItems.js

@@ -1,80 +0,0 @@
-const tariffTemplates = require('../tariffData.json')
-
-module.exports = async function(pg, items) {
-  await checkForPreviousItems(pg);
-
-  const {itemCounts, itemImages} = await createItems(pg, items);
-
-  await createShipment(pg, itemCounts);
-
-  return itemImages.map(({uuid, name, images}) => ({
-    uuid,
-    name,
-    images: images.map(image => image.replace(/^thumb\//, ''))
-  }))
-}
-
-async function checkForPreviousItems(pg) {
-  const items = await pg.item.findAll();
-
-  if(items.length)
-    throw new Error(`Cannot migrate items - ${items.length} items already exist!`)
-}
-
-async function createItems(pg, items) {
-  const itemCounts = [];
-  const itemImages = [];
-
-  for (const item of items) {
-    let tariffData = null;
-
-    if(item.tags.includes('fauxpaws'))
-      tariffData = tariffTemplates.sock;
-
-    if(item.tags.includes('tinyequine'))
-      tariffData = tariffTemplates.sock;
-
-    if(!tariffData) {
-      console.warn(`  Skipping item ${item.name} because of missing tariff data`)
-      continue;
-    }
-
-    const newItem = await pg.item.create(
-      item.name,
-      item.urlslug,
-      item.description
-        .replace(/<(\/|)p>/g, '')
-        .replace('&nbsp;', ' ')
-        .trim(),
-      item.price * 100,
-      true,
-      tariffData.hs_tariff_number,
-      tariffData.customs_description,
-      tariffData.origin_country,
-      tariffData.weight
-    )
-
-    itemCounts.push({
-      uuid: newItem.uuid,
-      count: item.numberInStock
-    })
-
-    itemImages.push({
-      uuid: newItem.uuid,
-      name: newItem.name,
-      images: [
-        item.productImage,
-        ...item.images
-      ]
-    })
-  }
-
-  return {itemCounts, itemImages};
-}
-
-async function createShipment(pg, itemCounts) {
-  return await pg.shipment.createShipment(
-    "Initial stock from db import",
-    itemCounts.filter(({count}) => count > 0)
-  );
-}

import/tasks/createUsers.js

@@ -1,35 +0,0 @@
-module.exports = async function(pg, users) {
-  let adminUser
-
-  for (const user of users) {
-    const existing = await pg.user.findByEmail(user.email)
-
-    if(existing) {
-      console.warn("  Warning: duplicate user with email " + user.email)
-      continue;
-    }
-
-    const newUser = await pg.user.import(user.email, user.password);
-
-    if (user.emailConfirmed) {
-      await pg.user.markEmailVerified(newUser.uuid)
-    }
-
-    if (user.isAdmin) {
-      await pg.user.makeAdmin(newUser.uuid)
-    }
-
-    if (user.isAdmin && !adminUser) {
-      adminUser = newUser;
-    }
-
-    const creationDate = new Date(parseInt(user._id.toString().slice(0,8), 16)*1000)
-    await pg.user.updateRegistrationDate(newUser.uuid, creationDate, user.emailConfirmed)
-  }
-
-  if (!adminUser) {
-    throw new Error("Unable to find importing admin")
-  }
-
-  return adminUser
-}

import/tasks/uploadImages.js

@@ -1,18 +0,0 @@
-const axios = require('axios');
-const db = require('../../db');
-
-module.exports = async function(db, itemImages, adminUUID) {
-  for (const item of itemImages) {
-    const {uuid, name, images} = item;
-
-    console.log(`  Downloading images for item: ${name}`)
-
-    for (const imageName of images) {
-      const url = `https://societyofsocks.us/media/${imageName}`
-      console.log('    Downloading ' + url)
-      const response = await axios.get(url, { responseType: 'arraybuffer' })
-
-      await db.item.addImage(uuid, response.data, adminUUID);
-    }
-  }
-}

import/tasks/writeExcelSheet.js

@@ -1,173 +0,0 @@
-const ExcelJS = require('exceljs');
-const easypost = new (require('@easypost/api'))(process.env.EASYPOST_API_KEY);
-
-module.exports = async function(users, coupons, items, filename) {
-  const workbook = new ExcelJS.Workbook();
-  workbook.creator = 'Ashe Erickson'
-  workbook.lastModifiedBy = 'Ashe Erickson'
-  workbook.created = new Date()
-  workbook.modified = new Date()
-
-  const usersSheet = workbook.addWorksheet('Users', {views:[{state: 'frozen', xSplit: 0, ySplit:1}]})
-  const ordersSheet = workbook.addWorksheet('Orders', {views:[{state: 'frozen', xSplit: 0, ySplit:1}]})
-  const couponsSheet = workbook.addWorksheet('Coupons', {views:[{state: 'frozen', xSplit: 0, ySplit:1}]})
-
-  const couponCache = {};
-
-
-  // Write coupons
-  couponsSheet.columns = [
-    {header: 'Coupon Code', width: 20},
-    {header: 'Expires', width: 14},
-    {header: 'Flat discount', key: 'flat', width: 12},
-    {header: '% discount', key: 'percent', width: 10},
-    {header: '# Socks free', width: 12},
-    {header: 'Per sock discount', key: 'perSock', width: 18},
-    {header: 'Free Shipping', width: 15},
-    {header: '# Allowed Uses', width: 18},
-    {header: '# Times Used', width: 18},
-  ]
-
-  for (const coupon of coupons) {
-    const row = couponsSheet.addRow([
-      coupon.code,
-      new Date(coupon.expires * 1000),
-      formatMoney(coupon.flatDiscount),
-      coupon.percentDiscount ? coupon.percentDiscount / 100 : '',
-      coupon.socksFree || '',
-      formatMoney(coupon.perSockDiscount),
-      coupon.freeShipping ? 'Yes' : 'No',
-      coupon.numAllowedUses,
-      coupon.uses.length
-    ])
-
-    row.eachCell(cell => {cell.alignment = {horizontal: 'left' }})
-
-    couponCache[coupon._id.toString()] = {
-      code: coupon.code,
-      row: row.number
-    }
-  }
-
-  // Write users + orders
-  usersSheet.columns = [
-    {header: 'Email', width: 40},
-    {header: 'Email Confirmed', width: 10},
-    {header: 'Registered', width: 20},
-    {header: 'Last Login', width: 20},
-    {header: 'Purchases', width: 20},
-    {header: 'Admin', width: 20},
-  ]
-
-  ordersSheet.columns = [
-    {header: 'User', width: 40},
-    {header: 'Purchase Date', width: 14},
-    {header: 'Shipment Date', width: 14},
-    {header: 'Items', width: 60},
-    {header: 'Item Subtotal', key: 'itemCost', width: 20},
-    {header: 'Shipping Estimate', key: 'shippingEst', width: 20},
-    {header: 'Shipping Actual', key: 'shippingCost', width: 20},
-    {header: 'Total Paid', key: 'totalCost', width: 20},
-    {header: 'Coupon', key: 'coupon', width: 20},
-    {header: 'Tracking Code', width: 20},
-    {header: 'Address', width: 20},
-  ]
-
-  const usersWithPurchases = users.filter(user => user.purchases?.length);
-  usersWithPurchases.sort((a,b) => (a.email || 'zzzz').localeCompare(b.email || 'zzzz'))
-
-  let userNum = 0;
-  for(const user of usersWithPurchases) {
-    if (user.purchases?.length < 1) continue;
-    userNum++
-
-    const creationDate = new Date(parseInt(user._id.toString().slice(0,8), 16)*1000)
-    const userID = user.email || user._id.toString();
-    const firstOrderRow = ordersSheet.rowCount + 1;
-    const userRow = usersSheet.rowCount + 1;
-
-
-    console.log(`  (${userNum}/${usersWithPurchases.length}) Retrieving shipment + address info for orders by user ${userID}`)
-
-    for(const purchase of user.purchases) {
-      const shipmentPromise = easypost.Shipment.retrieve(purchase.shipment);
-      const addressPromise = easypost.Address.retrieve(purchase.address);
-
-      try {
-        purchase.shipment = await shipmentPromise;
-        purchase.address = await addressPromise;
-      } catch {
-        console.warn(`    Unable to retrieve info for order ${purchase._id}`)
-      }
-
-      if (purchase.shipment?.tracker?.id) {
-        easypost.Tracker.retrieve(purchase.shipment.tracker.id).then(tracker => {
-          purchase.shipment.tracker = tracker
-        }).catch(() => {})
-      }
-
-      const coupon = couponCache[purchase.coupon]
-      const trackingCode = purchase.shipment?.tracking_code || purchase.trackingCode
-      const trackingLink = purchase.shipment?.tracker?.public_url || `https://tools.usps.com/go/TrackConfirmAction_input?qtc_tLabels1=${trackingCode}`
-
-      const row = ordersSheet.addRow([
-        user.email ? {formula: `=HYPERLINK("#'Users'!A${userRow}", "${userID}")`} : userID,
-        purchase.purchaseTime ? new Date(purchase.purchaseTime) : '',
-        purchase.shippedOn ? new Date(purchase.shippedOn) : '',
-        purchase.items.map(id => items.find(item => item._id.toString() === id.toString()).urlslug).join(', '),
-        formatMoney(purchase.sockPrice),
-        formatMoney(purchase.shippingEstimate),
-        formatMoney(purchase.shipment?.selected_rate?.rate),
-        formatMoney(purchase.totalPrice),
-        coupon ? {formula: `=HYPERLINK("#'Coupons'!A${coupon.row}","${coupon.code}")`} : '',
-        trackingCode ? {formula: `=HYPERLINK("${trackingLink}", "${trackingCode}")`} : "",
-        formatAddress(purchase.address)
-      ])
-
-      row.eachCell(cell => {cell.alignment = {horizontal: 'left' }})
-    }
-
-    if (!user.email) continue;
-
-    const row = usersSheet.addRow([
-      userID,
-      user.emailConfirmed ? 'Yes' : 'No',
-      creationDate,
-      user.lastLogin ? new Date(user.lastLogin) : '',
-      {formula: `=HYPERLINK("#'Orders'!A${firstOrderRow}", "${user.purchases.length} purchase${user.purchases.length > 1 ? "s" : ""}")`},
-      user.isAdmin ? 'Yes' : 'No'
-    ])
-
-    row.eachCell(cell => {cell.alignment = {horizontal: 'left' }})
-  }
-
-  ordersSheet.getColumn('itemCost').eachCell(cell => {cell.numFmt = '$0.00'})
-  ordersSheet.getColumn('shippingEst').eachCell(cell => {cell.numFmt = '$0.00'})
-  ordersSheet.getColumn('shippingCost').eachCell(cell => {cell.numFmt = '$0.00'})
-  ordersSheet.getColumn('totalCost').eachCell(cell => {cell.numFmt = '$0.00'})
-
-  couponsSheet.getColumn('flat').eachCell(cell => {cell.numFmt = '$0.00'})
-  couponsSheet.getColumn('percent').eachCell(cell => {cell.numFmt = '0%'})
-  couponsSheet.getColumn('perSock').eachCell(cell => {cell.numFmt = '$0.00'})
-
-  await workbook.xlsx.writeFile(filename);
-}
-
-function formatMoney(value) {
-  if(value === undefined) return ''
-  if(value === '') return ''
-  if(value === 0) return ''
-
-  if(typeof value === 'string')
-    return parseFloat(value);
-
-
-  return value
-}
-
-function formatAddress(address) {
-  if(typeof address !== 'object')
-    return address
-
-  return `${address.name || ""} | ${address.company ? address.company + "| " : ""}${address.street1} | ${address.street2 ? address.street2 + "| " : ""}${address.city} | ${address.state} | ${address.country} | ${address.zip}`
-}