Optimization for image thumbnails

main
Ashelyn Dawn 3 years ago
parent 55ea2dbf72
commit ce10c3e861

1
.gitignore vendored

@ -1,3 +1,4 @@
.next/ .next/
node_modules/ node_modules/
.env .env
cache/

@ -1,14 +1,63 @@
const router = require('express-promise-router')() const router = require('express-promise-router')()
const db = require('../db') const db = require('../db')
const ensureAdmin = require('./middleware/ensureAdmin') const ensureAdmin = require('./middleware/ensureAdmin')
const fs = require('fs');
const path = require('path');
const cacheRoot = path.join(__dirname, '../cache/images')
fs.promises.mkdir(cacheRoot, {recursive: true})
.catch(() => console.error("Could not create image cache dir: " + cacheRoot));
const memCache = {};
async function writeImageFile(uuid, size, data) {
if(size !== 'thumb' && size !== 'large')
throw new Error(`Unknown image size ${size}`);
await fs.promises.writeFile(path.join(cacheRoot, `${uuid}-${size}`), data);
}
async function getImage(uuid, size) {
// If we've already stored a mime type, that means we've written the file to disk
if(memCache[`${uuid}-${size}`]?.mime_type) {
return {
file: fs.createReadStream(path.join(cacheRoot, `${uuid}-${size}`)),
mime_type: memCache[`${uuid}-${size}`].mime_type
}
}
// If we've stored a promise, that means we're currently retrieving it from the db
if(memCache[`${uuid}-${size}`]?.dbPromise) {
return await memCache[`${uuid}-${size}`]?.dbPromise;
}
// Otherwise, retrieve it from the DB, and then write it to disk
const record = memCache[`${uuid}-${size}`] = {};
record.dbPromise = db.item.getImage(uuid, size);
const image = await record.dbPromise;
await writeImageFile(uuid, size, image.file);
record.mime_type = image.mime_type;
delete record.dbPromise;
return image;
}
router.get('/:uuid/:size', async (req, res) => { router.get('/:uuid/:size', async (req, res) => {
const image = await db.item.getImage(req.params.uuid, req.params.size) const image = await getImage(req.params.uuid, req.params.size)
const cacheSeconds = 60 * 60 * 24; const cacheSeconds = 60 * 60 * 24;
res.set('Cache-Control', cacheSeconds); res.set('Cache-Control', cacheSeconds);
res.set('Content-Type', image.mime_type) res.set('Content-Type', image.mime_type)
res.end(image.file)
if(Buffer.isBuffer(image.file))
res.end(image.file)
else if (typeof image.file.pipe === 'function')
image.file.pipe(res)
else
throw new Error("Unable to send file to user");
}) })
router.post('/:uuid/featured', ensureAdmin, async (req, res) => { router.post('/:uuid/featured', ensureAdmin, async (req, res) => {

@ -74,6 +74,8 @@
.card img{ .card img{
width:80%; width:80%;
aspect-ratio: 4 / 3;
object-fit: cover;
padding:10px; padding:10px;
background: #d4d4fb; background: #d4d4fb;
border: solid 1px #dec8e2; border: solid 1px #dec8e2;

3
import/.gitignore vendored

@ -1,3 +0,0 @@
users.json
node_modules/
datafile.xlsx

@ -1,45 +0,0 @@
#!/usr/bin/env node
const path = require('path')
require('dotenv').config({path: path.join(__dirname, '../.env')})
const pg = require('../db/pg')
const newDB = require('../db')
const oldDB = require('./mongo')('mongodb://localhost/sos')
const createItems = require('./tasks/createItems')
const createUsers = require('./tasks/createUsers')
const uploadImages = require('./tasks/uploadImages')
const saveExcelDataFile = require('./tasks/writeExcelSheet')
async function doMigration() {
const items = await oldDB.sock.find().lean().exec()
const allUsers = await oldDB.user.find().populate('purchases').lean().exec()
const carts = allUsers.map(u => u.purchases).flat()
const coupons = await oldDB.coupon.find().lean().exec()
const registeredUsers = allUsers.filter(user => user.email)
console.log(`Loaded ${carts.length} purchases and ${registeredUsers.length} users`)
console.log(`Inserting ${registeredUsers.length} users into the database`)
const importAdmin = await createUsers(newDB, registeredUsers);
console.log(` Found user account ${importAdmin.uuid} (${importAdmin.email}) to attribute file uploads to`)
console.log(`\nInserting ${items.length} items into database`)
const itemImages = await createItems(newDB, items);
const numImages = itemImages.map(({images}) => images.length).reduce((a,b) => b+a, 0)
console.log(`\nImporting ${numImages} images into database`)
await uploadImages(newDB, itemImages, importAdmin.uuid)
console.log('\nWriting Excel data file')
await saveExcelDataFile(allUsers, coupons, items, path.join(__dirname, './datafile.xlsx'))
}
doMigration()
.catch(console.error)
.finally(() => {
pg.end()
oldDB._connection.close()
})

@ -1,24 +0,0 @@
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
module.exports.schema = new Schema({
user: {type: Schema.Types.ObjectId, ref: 'User'},
items: [{type: Schema.Types.ObjectId, required: true, ref: 'Sock'}],
purchased: {type: Schema.Types.Mixed, enum: [true, false, 'refunded'], default: false, required: true},
purchaseTime: {type: Number},
shipped: {type: Boolean, required: true, default: false},
shippedOn: {type: Number},
address: {type: String}, // Easypost id
shipment: {type: String}, // Easypost id
shipmentMeasured: {type: Boolean, default: false},
sockPrice: {type: Number},
totalPrice: {type: Number},
shippingEstimate: {type: Number},
coupon: {type: Schema.Types.ObjectId, required: false, ref: 'Coupon'},
trackingCode: {type: String},
needsCustoms: {type: Boolean, default: false}
}, {
usePushEach: true
});
module.exports.model = mongoose.model('Cart', module.exports.schema);

@ -1,20 +0,0 @@
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
module.exports.schema = new Schema({
code: {type: String, required: true, unique: true, index: true},
numAllowedUses: {type: Number, required: true, default: 1},
uses: [{type: Schema.Types.ObjectId, required: true, ref: 'Cart'}],
expires: {type: Number, required: true, default: 0},
flatDiscount: {type: Number, required: true, default: 0},
percentDiscount: {type: Number, required: true, default:0},
socksFree: {type: Number, required: true, default: 0},
perSockDiscount: {type: Number, required: false, default: 0},
freeShipping: {type: Boolean, required: true, default: false}
}, {
usePushEach: true
});
module.exports.model = mongoose.model('Coupon', module.exports.schema);

@ -1,31 +0,0 @@
const mongoose = require('mongoose')
mongoose.Promise = Promise
const models = {
Sock: require('./sock.js').schema,
Media: require('./media.js').schema,
User: require('./user.js').schema,
Cart: require('./cart.js').schema,
Coupon: require('./coupon.js').schema,
}
module.exports = function connect(url) {
const connection = mongoose.createConnection(url)
connection.on('error', console.error.bind(console, 'connection error:'));
var sock = connection.model('Sock',models.Sock, 'socks');
var media = connection.model('Media',models.Media, 'medias');
var user = connection.model('User',models.User, 'users');
var cart = connection.model('Cart',models.Cart, 'carts');
var coupon = connection.model('Coupon',models.Coupon, 'coupons');
return {
sock,
media,
user,
cart,
coupon,
_connection: connection
}
}

@ -1,18 +0,0 @@
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var MediaSchema = new Schema({
name: {type: String, required: true},
urlslug: {type: String, required: true, unique: true},
uploaded: {type: Number, required: true},
filename: {type: String, required: true, unique: true},
thumbnail: {type: String},
mimetype: {type: String, required: true},
width: {type:Number,required:true},
height: {type:Number,required:true}
});
var Media = mongoose.model('Media', MediaSchema);
module.exports.model = Media;
module.exports.schema = MediaSchema

@ -1,20 +0,0 @@
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
module.exports.schema = new Schema({
name: {type: String, required: true},
urlslug: {type: String, required: true, lowercase: true, unique: true},
productImage: {type: String, required: true},
description: {type: String, required: true},
numberInStock: {type: Number},
price: {type: Number},
images: [String], //Does not include productImage
tags: [String],
publishTime:{type: Number, required: true},
expireTime:{type: Number, required: true},
preorderDate: {type: Number, default: null}
});
module.exports.schema.index({name: 'text', description: 'text'});
module.exports.model = mongoose.model('Sock', module.exports.schema);

@ -1,19 +0,0 @@
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
module.exports.schema = new Schema({
email: {type: String},
emailConfirmed: {type: Boolean, default: false},
emailPin: {type: String},
password: {type: String},
isAdmin: {type: Boolean, default: false},
cart: {type: Schema.Types.ObjectId, ref: 'Cart'},
purchases: [{type: Schema.Types.ObjectId, ref: 'Cart'}],
credit: {type: Number, default: 0},//Credit in cents because non-integers are confusing?
savedAddress: {type: String},
lastLogin: {type: Number, required: true}
}, {
usePushEach: true
});
module.exports.model = mongoose.model('User', module.exports.schema);

File diff suppressed because it is too large Load Diff

@ -1,8 +0,0 @@
{
"name": "sos-import",
"version": "0.0.1",
"dependencies": {
"exceljs": "^4.2.1",
"mongoose": "^4.4.16"
}
}

@ -1,8 +0,0 @@
{
"sock": {
"hs_tariff_number": "611595",
"customs_description": "A pair of socks",
"origin_country": "CN",
"weight": 2
}
}

@ -1,80 +0,0 @@
const tariffTemplates = require('../tariffData.json')
module.exports = async function(pg, items) {
await checkForPreviousItems(pg);
const {itemCounts, itemImages} = await createItems(pg, items);
await createShipment(pg, itemCounts);
return itemImages.map(({uuid, name, images}) => ({
uuid,
name,
images: images.map(image => image.replace(/^thumb\//, ''))
}))
}
async function checkForPreviousItems(pg) {
const items = await pg.item.findAll();
if(items.length)
throw new Error(`Cannot migrate items - ${items.length} items already exist!`)
}
async function createItems(pg, items) {
const itemCounts = [];
const itemImages = [];
for (const item of items) {
let tariffData = null;
if(item.tags.includes('fauxpaws'))
tariffData = tariffTemplates.sock;
if(item.tags.includes('tinyequine'))
tariffData = tariffTemplates.sock;
if(!tariffData) {
console.warn(` Skipping item ${item.name} because of missing tariff data`)
continue;
}
const newItem = await pg.item.create(
item.name,
item.urlslug,
item.description
.replace(/<(\/|)p>/g, '')
.replace('&nbsp;', ' ')
.trim(),
item.price * 100,
true,
tariffData.hs_tariff_number,
tariffData.customs_description,
tariffData.origin_country,
tariffData.weight
)
itemCounts.push({
uuid: newItem.uuid,
count: item.numberInStock
})
itemImages.push({
uuid: newItem.uuid,
name: newItem.name,
images: [
item.productImage,
...item.images
]
})
}
return {itemCounts, itemImages};
}
async function createShipment(pg, itemCounts) {
return await pg.shipment.createShipment(
"Initial stock from db import",
itemCounts.filter(({count}) => count > 0)
);
}

@ -1,35 +0,0 @@
module.exports = async function(pg, users) {
let adminUser
for (const user of users) {
const existing = await pg.user.findByEmail(user.email)
if(existing) {
console.warn(" Warning: duplicate user with email " + user.email)
continue;
}
const newUser = await pg.user.import(user.email, user.password);
if (user.emailConfirmed) {
await pg.user.markEmailVerified(newUser.uuid)
}
if (user.isAdmin) {
await pg.user.makeAdmin(newUser.uuid)
}
if (user.isAdmin && !adminUser) {
adminUser = newUser;
}
const creationDate = new Date(parseInt(user._id.toString().slice(0,8), 16)*1000)
await pg.user.updateRegistrationDate(newUser.uuid, creationDate, user.emailConfirmed)
}
if (!adminUser) {
throw new Error("Unable to find importing admin")
}
return adminUser
}

@ -1,18 +0,0 @@
const axios = require('axios');
const db = require('../../db');
module.exports = async function(db, itemImages, adminUUID) {
for (const item of itemImages) {
const {uuid, name, images} = item;
console.log(` Downloading images for item: ${name}`)
for (const imageName of images) {
const url = `https://societyofsocks.us/media/${imageName}`
console.log(' Downloading ' + url)
const response = await axios.get(url, { responseType: 'arraybuffer' })
await db.item.addImage(uuid, response.data, adminUUID);
}
}
}

@ -1,173 +0,0 @@
const ExcelJS = require('exceljs');
const easypost = new (require('@easypost/api'))(process.env.EASYPOST_API_KEY);
module.exports = async function(users, coupons, items, filename) {
const workbook = new ExcelJS.Workbook();
workbook.creator = 'Ashe Erickson'
workbook.lastModifiedBy = 'Ashe Erickson'
workbook.created = new Date()
workbook.modified = new Date()
const usersSheet = workbook.addWorksheet('Users', {views:[{state: 'frozen', xSplit: 0, ySplit:1}]})
const ordersSheet = workbook.addWorksheet('Orders', {views:[{state: 'frozen', xSplit: 0, ySplit:1}]})
const couponsSheet = workbook.addWorksheet('Coupons', {views:[{state: 'frozen', xSplit: 0, ySplit:1}]})
const couponCache = {};
// Write coupons
couponsSheet.columns = [
{header: 'Coupon Code', width: 20},
{header: 'Expires', width: 14},
{header: 'Flat discount', key: 'flat', width: 12},
{header: '% discount', key: 'percent', width: 10},
{header: '# Socks free', width: 12},
{header: 'Per sock discount', key: 'perSock', width: 18},
{header: 'Free Shipping', width: 15},
{header: '# Allowed Uses', width: 18},
{header: '# Times Used', width: 18},
]
for (const coupon of coupons) {
const row = couponsSheet.addRow([
coupon.code,
new Date(coupon.expires * 1000),
formatMoney(coupon.flatDiscount),
coupon.percentDiscount ? coupon.percentDiscount / 100 : '',
coupon.socksFree || '',
formatMoney(coupon.perSockDiscount),
coupon.freeShipping ? 'Yes' : 'No',
coupon.numAllowedUses,
coupon.uses.length
])
row.eachCell(cell => {cell.alignment = {horizontal: 'left' }})
couponCache[coupon._id.toString()] = {
code: coupon.code,
row: row.number
}
}
// Write users + orders
usersSheet.columns = [
{header: 'Email', width: 40},
{header: 'Email Confirmed', width: 10},
{header: 'Registered', width: 20},
{header: 'Last Login', width: 20},
{header: 'Purchases', width: 20},
{header: 'Admin', width: 20},
]
ordersSheet.columns = [
{header: 'User', width: 40},
{header: 'Purchase Date', width: 14},
{header: 'Shipment Date', width: 14},
{header: 'Items', width: 60},
{header: 'Item Subtotal', key: 'itemCost', width: 20},
{header: 'Shipping Estimate', key: 'shippingEst', width: 20},
{header: 'Shipping Actual', key: 'shippingCost', width: 20},
{header: 'Total Paid', key: 'totalCost', width: 20},
{header: 'Coupon', key: 'coupon', width: 20},
{header: 'Tracking Code', width: 20},
{header: 'Address', width: 20},
]
const usersWithPurchases = users.filter(user => user.purchases?.length);
usersWithPurchases.sort((a,b) => (a.email || 'zzzz').localeCompare(b.email || 'zzzz'))
let userNum = 0;
for(const user of usersWithPurchases) {
if (user.purchases?.length < 1) continue;
userNum++
const creationDate = new Date(parseInt(user._id.toString().slice(0,8), 16)*1000)
const userID = user.email || user._id.toString();
const firstOrderRow = ordersSheet.rowCount + 1;
const userRow = usersSheet.rowCount + 1;
console.log(` (${userNum}/${usersWithPurchases.length}) Retrieving shipment + address info for orders by user ${userID}`)
for(const purchase of user.purchases) {
const shipmentPromise = easypost.Shipment.retrieve(purchase.shipment);
const addressPromise = easypost.Address.retrieve(purchase.address);
try {
purchase.shipment = await shipmentPromise;
purchase.address = await addressPromise;
} catch {
console.warn(` Unable to retrieve info for order ${purchase._id}`)
}
if (purchase.shipment?.tracker?.id) {
easypost.Tracker.retrieve(purchase.shipment.tracker.id).then(tracker => {
purchase.shipment.tracker = tracker
}).catch(() => {})
}
const coupon = couponCache[purchase.coupon]
const trackingCode = purchase.shipment?.tracking_code || purchase.trackingCode
const trackingLink = purchase.shipment?.tracker?.public_url || `https://tools.usps.com/go/TrackConfirmAction_input?qtc_tLabels1=${trackingCode}`
const row = ordersSheet.addRow([
user.email ? {formula: `=HYPERLINK("#'Users'!A${userRow}", "${userID}")`} : userID,
purchase.purchaseTime ? new Date(purchase.purchaseTime) : '',
purchase.shippedOn ? new Date(purchase.shippedOn) : '',
purchase.items.map(id => items.find(item => item._id.toString() === id.toString()).urlslug).join(', '),
formatMoney(purchase.sockPrice),
formatMoney(purchase.shippingEstimate),
formatMoney(purchase.shipment?.selected_rate?.rate),
formatMoney(purchase.totalPrice),
coupon ? {formula: `=HYPERLINK("#'Coupons'!A${coupon.row}","${coupon.code}")`} : '',
trackingCode ? {formula: `=HYPERLINK("${trackingLink}", "${trackingCode}")`} : "",
formatAddress(purchase.address)
])
row.eachCell(cell => {cell.alignment = {horizontal: 'left' }})
}
if (!user.email) continue;
const row = usersSheet.addRow([
userID,
user.emailConfirmed ? 'Yes' : 'No',
creationDate,
user.lastLogin ? new Date(user.lastLogin) : '',
{formula: `=HYPERLINK("#'Orders'!A${firstOrderRow}", "${user.purchases.length} purchase${user.purchases.length > 1 ? "s" : ""}")`},
user.isAdmin ? 'Yes' : 'No'
])
row.eachCell(cell => {cell.alignment = {horizontal: 'left' }})
}
ordersSheet.getColumn('itemCost').eachCell(cell => {cell.numFmt = '$0.00'})
ordersSheet.getColumn('shippingEst').eachCell(cell => {cell.numFmt = '$0.00'})
ordersSheet.getColumn('shippingCost').eachCell(cell => {cell.numFmt = '$0.00'})
ordersSheet.getColumn('totalCost').eachCell(cell => {cell.numFmt = '$0.00'})
couponsSheet.getColumn('flat').eachCell(cell => {cell.numFmt = '$0.00'})
couponsSheet.getColumn('percent').eachCell(cell => {cell.numFmt = '0%'})
couponsSheet.getColumn('perSock').eachCell(cell => {cell.numFmt = '$0.00'})
await workbook.xlsx.writeFile(filename);
}
function formatMoney(value) {
if(value === undefined) return ''
if(value === '') return ''
if(value === 0) return ''
if(typeof value === 'string')
return parseFloat(value);
return value
}
function formatAddress(address) {
if(typeof address !== 'object')
return address
return `${address.name || ""} | ${address.company ? address.company + "| " : ""}${address.street1} | ${address.street2 ? address.street2 + "| " : ""}${address.city} | ${address.state} | ${address.country} | ${address.zip}`
}
Loading…
Cancel
Save