Optimization for image thumbnails
parent
55ea2dbf72
commit
ce10c3e861
@ -1,3 +1,4 @@
|
||||
.next/
|
||||
node_modules/
|
||||
.env
|
||||
cache/
|
@ -1,3 +0,0 @@
|
||||
users.json
|
||||
node_modules/
|
||||
datafile.xlsx
|
@ -1,45 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
const path = require('path')
|
||||
require('dotenv').config({path: path.join(__dirname, '../.env')})
|
||||
|
||||
const pg = require('../db/pg')
|
||||
const newDB = require('../db')
|
||||
const oldDB = require('./mongo')('mongodb://localhost/sos')
|
||||
|
||||
const createItems = require('./tasks/createItems')
|
||||
const createUsers = require('./tasks/createUsers')
|
||||
const uploadImages = require('./tasks/uploadImages')
|
||||
const saveExcelDataFile = require('./tasks/writeExcelSheet')
|
||||
|
||||
async function doMigration() {
|
||||
const items = await oldDB.sock.find().lean().exec()
|
||||
const allUsers = await oldDB.user.find().populate('purchases').lean().exec()
|
||||
const carts = allUsers.map(u => u.purchases).flat()
|
||||
const coupons = await oldDB.coupon.find().lean().exec()
|
||||
|
||||
const registeredUsers = allUsers.filter(user => user.email)
|
||||
|
||||
console.log(`Loaded ${carts.length} purchases and ${registeredUsers.length} users`)
|
||||
|
||||
console.log(`Inserting ${registeredUsers.length} users into the database`)
|
||||
const importAdmin = await createUsers(newDB, registeredUsers);
|
||||
console.log(` Found user account ${importAdmin.uuid} (${importAdmin.email}) to attribute file uploads to`)
|
||||
|
||||
console.log(`\nInserting ${items.length} items into database`)
|
||||
const itemImages = await createItems(newDB, items);
|
||||
|
||||
const numImages = itemImages.map(({images}) => images.length).reduce((a,b) => b+a, 0)
|
||||
console.log(`\nImporting ${numImages} images into database`)
|
||||
await uploadImages(newDB, itemImages, importAdmin.uuid)
|
||||
|
||||
console.log('\nWriting Excel data file')
|
||||
await saveExcelDataFile(allUsers, coupons, items, path.join(__dirname, './datafile.xlsx'))
|
||||
}
|
||||
|
||||
doMigration()
|
||||
.catch(console.error)
|
||||
.finally(() => {
|
||||
pg.end()
|
||||
oldDB._connection.close()
|
||||
})
|
||||
|
@ -1,24 +0,0 @@
|
||||
var mongoose = require('mongoose');
|
||||
var Schema = mongoose.Schema;
|
||||
|
||||
module.exports.schema = new Schema({
|
||||
user: {type: Schema.Types.ObjectId, ref: 'User'},
|
||||
items: [{type: Schema.Types.ObjectId, required: true, ref: 'Sock'}],
|
||||
purchased: {type: Schema.Types.Mixed, enum: [true, false, 'refunded'], default: false, required: true},
|
||||
purchaseTime: {type: Number},
|
||||
shipped: {type: Boolean, required: true, default: false},
|
||||
shippedOn: {type: Number},
|
||||
address: {type: String}, // Easypost id
|
||||
shipment: {type: String}, // Easypost id
|
||||
shipmentMeasured: {type: Boolean, default: false},
|
||||
sockPrice: {type: Number},
|
||||
totalPrice: {type: Number},
|
||||
shippingEstimate: {type: Number},
|
||||
coupon: {type: Schema.Types.ObjectId, required: false, ref: 'Coupon'},
|
||||
trackingCode: {type: String},
|
||||
needsCustoms: {type: Boolean, default: false}
|
||||
}, {
|
||||
usePushEach: true
|
||||
});
|
||||
|
||||
module.exports.model = mongoose.model('Cart', module.exports.schema);
|
@ -1,20 +0,0 @@
|
||||
var mongoose = require('mongoose');
|
||||
var Schema = mongoose.Schema;
|
||||
|
||||
module.exports.schema = new Schema({
|
||||
code: {type: String, required: true, unique: true, index: true},
|
||||
|
||||
numAllowedUses: {type: Number, required: true, default: 1},
|
||||
uses: [{type: Schema.Types.ObjectId, required: true, ref: 'Cart'}],
|
||||
expires: {type: Number, required: true, default: 0},
|
||||
|
||||
flatDiscount: {type: Number, required: true, default: 0},
|
||||
percentDiscount: {type: Number, required: true, default:0},
|
||||
socksFree: {type: Number, required: true, default: 0},
|
||||
perSockDiscount: {type: Number, required: false, default: 0},
|
||||
freeShipping: {type: Boolean, required: true, default: false}
|
||||
}, {
|
||||
usePushEach: true
|
||||
});
|
||||
|
||||
module.exports.model = mongoose.model('Coupon', module.exports.schema);
|
@ -1,31 +0,0 @@
|
||||
const mongoose = require('mongoose')
|
||||
mongoose.Promise = Promise
|
||||
|
||||
const models = {
|
||||
Sock: require('./sock.js').schema,
|
||||
Media: require('./media.js').schema,
|
||||
User: require('./user.js').schema,
|
||||
Cart: require('./cart.js').schema,
|
||||
Coupon: require('./coupon.js').schema,
|
||||
}
|
||||
|
||||
module.exports = function connect(url) {
|
||||
const connection = mongoose.createConnection(url)
|
||||
|
||||
connection.on('error', console.error.bind(console, 'connection error:'));
|
||||
|
||||
var sock = connection.model('Sock',models.Sock, 'socks');
|
||||
var media = connection.model('Media',models.Media, 'medias');
|
||||
var user = connection.model('User',models.User, 'users');
|
||||
var cart = connection.model('Cart',models.Cart, 'carts');
|
||||
var coupon = connection.model('Coupon',models.Coupon, 'coupons');
|
||||
|
||||
return {
|
||||
sock,
|
||||
media,
|
||||
user,
|
||||
cart,
|
||||
coupon,
|
||||
_connection: connection
|
||||
}
|
||||
}
|
@ -1,18 +0,0 @@
|
||||
var mongoose = require('mongoose');
|
||||
var Schema = mongoose.Schema;
|
||||
|
||||
var MediaSchema = new Schema({
|
||||
name: {type: String, required: true},
|
||||
urlslug: {type: String, required: true, unique: true},
|
||||
uploaded: {type: Number, required: true},
|
||||
filename: {type: String, required: true, unique: true},
|
||||
thumbnail: {type: String},
|
||||
mimetype: {type: String, required: true},
|
||||
width: {type:Number,required:true},
|
||||
height: {type:Number,required:true}
|
||||
});
|
||||
|
||||
var Media = mongoose.model('Media', MediaSchema);
|
||||
|
||||
module.exports.model = Media;
|
||||
module.exports.schema = MediaSchema
|
@ -1,20 +0,0 @@
|
||||
var mongoose = require('mongoose');
|
||||
var Schema = mongoose.Schema;
|
||||
|
||||
module.exports.schema = new Schema({
|
||||
name: {type: String, required: true},
|
||||
urlslug: {type: String, required: true, lowercase: true, unique: true},
|
||||
productImage: {type: String, required: true},
|
||||
description: {type: String, required: true},
|
||||
numberInStock: {type: Number},
|
||||
price: {type: Number},
|
||||
images: [String], //Does not include productImage
|
||||
tags: [String],
|
||||
publishTime:{type: Number, required: true},
|
||||
expireTime:{type: Number, required: true},
|
||||
preorderDate: {type: Number, default: null}
|
||||
});
|
||||
|
||||
module.exports.schema.index({name: 'text', description: 'text'});
|
||||
|
||||
module.exports.model = mongoose.model('Sock', module.exports.schema);
|
@ -1,19 +0,0 @@
|
||||
var mongoose = require('mongoose');
|
||||
var Schema = mongoose.Schema;
|
||||
|
||||
module.exports.schema = new Schema({
|
||||
email: {type: String},
|
||||
emailConfirmed: {type: Boolean, default: false},
|
||||
emailPin: {type: String},
|
||||
password: {type: String},
|
||||
isAdmin: {type: Boolean, default: false},
|
||||
cart: {type: Schema.Types.ObjectId, ref: 'Cart'},
|
||||
purchases: [{type: Schema.Types.ObjectId, ref: 'Cart'}],
|
||||
credit: {type: Number, default: 0},//Credit in cents because non-integers are confusing?
|
||||
savedAddress: {type: String},
|
||||
lastLogin: {type: Number, required: true}
|
||||
}, {
|
||||
usePushEach: true
|
||||
});
|
||||
|
||||
module.exports.model = mongoose.model('User', module.exports.schema);
|
File diff suppressed because it is too large
Load Diff
@ -1,8 +0,0 @@
|
||||
{
|
||||
"name": "sos-import",
|
||||
"version": "0.0.1",
|
||||
"dependencies": {
|
||||
"exceljs": "^4.2.1",
|
||||
"mongoose": "^4.4.16"
|
||||
}
|
||||
}
|
@ -1,8 +0,0 @@
|
||||
{
|
||||
"sock": {
|
||||
"hs_tariff_number": "611595",
|
||||
"customs_description": "A pair of socks",
|
||||
"origin_country": "CN",
|
||||
"weight": 2
|
||||
}
|
||||
}
|
@ -1,80 +0,0 @@
|
||||
const tariffTemplates = require('../tariffData.json')
|
||||
|
||||
module.exports = async function(pg, items) {
|
||||
await checkForPreviousItems(pg);
|
||||
|
||||
const {itemCounts, itemImages} = await createItems(pg, items);
|
||||
|
||||
await createShipment(pg, itemCounts);
|
||||
|
||||
return itemImages.map(({uuid, name, images}) => ({
|
||||
uuid,
|
||||
name,
|
||||
images: images.map(image => image.replace(/^thumb\//, ''))
|
||||
}))
|
||||
}
|
||||
|
||||
async function checkForPreviousItems(pg) {
|
||||
const items = await pg.item.findAll();
|
||||
|
||||
if(items.length)
|
||||
throw new Error(`Cannot migrate items - ${items.length} items already exist!`)
|
||||
}
|
||||
|
||||
async function createItems(pg, items) {
|
||||
const itemCounts = [];
|
||||
const itemImages = [];
|
||||
|
||||
for (const item of items) {
|
||||
let tariffData = null;
|
||||
|
||||
if(item.tags.includes('fauxpaws'))
|
||||
tariffData = tariffTemplates.sock;
|
||||
|
||||
if(item.tags.includes('tinyequine'))
|
||||
tariffData = tariffTemplates.sock;
|
||||
|
||||
if(!tariffData) {
|
||||
console.warn(` Skipping item ${item.name} because of missing tariff data`)
|
||||
continue;
|
||||
}
|
||||
|
||||
const newItem = await pg.item.create(
|
||||
item.name,
|
||||
item.urlslug,
|
||||
item.description
|
||||
.replace(/<(\/|)p>/g, '')
|
||||
.replace(' ', ' ')
|
||||
.trim(),
|
||||
item.price * 100,
|
||||
true,
|
||||
tariffData.hs_tariff_number,
|
||||
tariffData.customs_description,
|
||||
tariffData.origin_country,
|
||||
tariffData.weight
|
||||
)
|
||||
|
||||
itemCounts.push({
|
||||
uuid: newItem.uuid,
|
||||
count: item.numberInStock
|
||||
})
|
||||
|
||||
itemImages.push({
|
||||
uuid: newItem.uuid,
|
||||
name: newItem.name,
|
||||
images: [
|
||||
item.productImage,
|
||||
...item.images
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
return {itemCounts, itemImages};
|
||||
}
|
||||
|
||||
async function createShipment(pg, itemCounts) {
|
||||
return await pg.shipment.createShipment(
|
||||
"Initial stock from db import",
|
||||
itemCounts.filter(({count}) => count > 0)
|
||||
);
|
||||
}
|
@ -1,35 +0,0 @@
|
||||
module.exports = async function(pg, users) {
|
||||
let adminUser
|
||||
|
||||
for (const user of users) {
|
||||
const existing = await pg.user.findByEmail(user.email)
|
||||
|
||||
if(existing) {
|
||||
console.warn(" Warning: duplicate user with email " + user.email)
|
||||
continue;
|
||||
}
|
||||
|
||||
const newUser = await pg.user.import(user.email, user.password);
|
||||
|
||||
if (user.emailConfirmed) {
|
||||
await pg.user.markEmailVerified(newUser.uuid)
|
||||
}
|
||||
|
||||
if (user.isAdmin) {
|
||||
await pg.user.makeAdmin(newUser.uuid)
|
||||
}
|
||||
|
||||
if (user.isAdmin && !adminUser) {
|
||||
adminUser = newUser;
|
||||
}
|
||||
|
||||
const creationDate = new Date(parseInt(user._id.toString().slice(0,8), 16)*1000)
|
||||
await pg.user.updateRegistrationDate(newUser.uuid, creationDate, user.emailConfirmed)
|
||||
}
|
||||
|
||||
if (!adminUser) {
|
||||
throw new Error("Unable to find importing admin")
|
||||
}
|
||||
|
||||
return adminUser
|
||||
}
|
@ -1,18 +0,0 @@
|
||||
const axios = require('axios');
|
||||
const db = require('../../db');
|
||||
|
||||
module.exports = async function(db, itemImages, adminUUID) {
|
||||
for (const item of itemImages) {
|
||||
const {uuid, name, images} = item;
|
||||
|
||||
console.log(` Downloading images for item: ${name}`)
|
||||
|
||||
for (const imageName of images) {
|
||||
const url = `https://societyofsocks.us/media/${imageName}`
|
||||
console.log(' Downloading ' + url)
|
||||
const response = await axios.get(url, { responseType: 'arraybuffer' })
|
||||
|
||||
await db.item.addImage(uuid, response.data, adminUUID);
|
||||
}
|
||||
}
|
||||
}
|
@ -1,173 +0,0 @@
|
||||
const ExcelJS = require('exceljs');
|
||||
const easypost = new (require('@easypost/api'))(process.env.EASYPOST_API_KEY);
|
||||
|
||||
module.exports = async function(users, coupons, items, filename) {
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
workbook.creator = 'Ashe Erickson'
|
||||
workbook.lastModifiedBy = 'Ashe Erickson'
|
||||
workbook.created = new Date()
|
||||
workbook.modified = new Date()
|
||||
|
||||
const usersSheet = workbook.addWorksheet('Users', {views:[{state: 'frozen', xSplit: 0, ySplit:1}]})
|
||||
const ordersSheet = workbook.addWorksheet('Orders', {views:[{state: 'frozen', xSplit: 0, ySplit:1}]})
|
||||
const couponsSheet = workbook.addWorksheet('Coupons', {views:[{state: 'frozen', xSplit: 0, ySplit:1}]})
|
||||
|
||||
const couponCache = {};
|
||||
|
||||
|
||||
// Write coupons
|
||||
couponsSheet.columns = [
|
||||
{header: 'Coupon Code', width: 20},
|
||||
{header: 'Expires', width: 14},
|
||||
{header: 'Flat discount', key: 'flat', width: 12},
|
||||
{header: '% discount', key: 'percent', width: 10},
|
||||
{header: '# Socks free', width: 12},
|
||||
{header: 'Per sock discount', key: 'perSock', width: 18},
|
||||
{header: 'Free Shipping', width: 15},
|
||||
{header: '# Allowed Uses', width: 18},
|
||||
{header: '# Times Used', width: 18},
|
||||
]
|
||||
|
||||
for (const coupon of coupons) {
|
||||
const row = couponsSheet.addRow([
|
||||
coupon.code,
|
||||
new Date(coupon.expires * 1000),
|
||||
formatMoney(coupon.flatDiscount),
|
||||
coupon.percentDiscount ? coupon.percentDiscount / 100 : '',
|
||||
coupon.socksFree || '',
|
||||
formatMoney(coupon.perSockDiscount),
|
||||
coupon.freeShipping ? 'Yes' : 'No',
|
||||
coupon.numAllowedUses,
|
||||
coupon.uses.length
|
||||
])
|
||||
|
||||
row.eachCell(cell => {cell.alignment = {horizontal: 'left' }})
|
||||
|
||||
couponCache[coupon._id.toString()] = {
|
||||
code: coupon.code,
|
||||
row: row.number
|
||||
}
|
||||
}
|
||||
|
||||
// Write users + orders
|
||||
usersSheet.columns = [
|
||||
{header: 'Email', width: 40},
|
||||
{header: 'Email Confirmed', width: 10},
|
||||
{header: 'Registered', width: 20},
|
||||
{header: 'Last Login', width: 20},
|
||||
{header: 'Purchases', width: 20},
|
||||
{header: 'Admin', width: 20},
|
||||
]
|
||||
|
||||
ordersSheet.columns = [
|
||||
{header: 'User', width: 40},
|
||||
{header: 'Purchase Date', width: 14},
|
||||
{header: 'Shipment Date', width: 14},
|
||||
{header: 'Items', width: 60},
|
||||
{header: 'Item Subtotal', key: 'itemCost', width: 20},
|
||||
{header: 'Shipping Estimate', key: 'shippingEst', width: 20},
|
||||
{header: 'Shipping Actual', key: 'shippingCost', width: 20},
|
||||
{header: 'Total Paid', key: 'totalCost', width: 20},
|
||||
{header: 'Coupon', key: 'coupon', width: 20},
|
||||
{header: 'Tracking Code', width: 20},
|
||||
{header: 'Address', width: 20},
|
||||
]
|
||||
|
||||
const usersWithPurchases = users.filter(user => user.purchases?.length);
|
||||
usersWithPurchases.sort((a,b) => (a.email || 'zzzz').localeCompare(b.email || 'zzzz'))
|
||||
|
||||
let userNum = 0;
|
||||
for(const user of usersWithPurchases) {
|
||||
if (user.purchases?.length < 1) continue;
|
||||
userNum++
|
||||
|
||||
const creationDate = new Date(parseInt(user._id.toString().slice(0,8), 16)*1000)
|
||||
const userID = user.email || user._id.toString();
|
||||
const firstOrderRow = ordersSheet.rowCount + 1;
|
||||
const userRow = usersSheet.rowCount + 1;
|
||||
|
||||
|
||||
console.log(` (${userNum}/${usersWithPurchases.length}) Retrieving shipment + address info for orders by user ${userID}`)
|
||||
|
||||
for(const purchase of user.purchases) {
|
||||
const shipmentPromise = easypost.Shipment.retrieve(purchase.shipment);
|
||||
const addressPromise = easypost.Address.retrieve(purchase.address);
|
||||
|
||||
try {
|
||||
purchase.shipment = await shipmentPromise;
|
||||
purchase.address = await addressPromise;
|
||||
} catch {
|
||||
console.warn(` Unable to retrieve info for order ${purchase._id}`)
|
||||
}
|
||||
|
||||
if (purchase.shipment?.tracker?.id) {
|
||||
easypost.Tracker.retrieve(purchase.shipment.tracker.id).then(tracker => {
|
||||
purchase.shipment.tracker = tracker
|
||||
}).catch(() => {})
|
||||
}
|
||||
|
||||
const coupon = couponCache[purchase.coupon]
|
||||
const trackingCode = purchase.shipment?.tracking_code || purchase.trackingCode
|
||||
const trackingLink = purchase.shipment?.tracker?.public_url || `https://tools.usps.com/go/TrackConfirmAction_input?qtc_tLabels1=${trackingCode}`
|
||||
|
||||
const row = ordersSheet.addRow([
|
||||
user.email ? {formula: `=HYPERLINK("#'Users'!A${userRow}", "${userID}")`} : userID,
|
||||
purchase.purchaseTime ? new Date(purchase.purchaseTime) : '',
|
||||
purchase.shippedOn ? new Date(purchase.shippedOn) : '',
|
||||
purchase.items.map(id => items.find(item => item._id.toString() === id.toString()).urlslug).join(', '),
|
||||
formatMoney(purchase.sockPrice),
|
||||
formatMoney(purchase.shippingEstimate),
|
||||
formatMoney(purchase.shipment?.selected_rate?.rate),
|
||||
formatMoney(purchase.totalPrice),
|
||||
coupon ? {formula: `=HYPERLINK("#'Coupons'!A${coupon.row}","${coupon.code}")`} : '',
|
||||
trackingCode ? {formula: `=HYPERLINK("${trackingLink}", "${trackingCode}")`} : "",
|
||||
formatAddress(purchase.address)
|
||||
])
|
||||
|
||||
row.eachCell(cell => {cell.alignment = {horizontal: 'left' }})
|
||||
}
|
||||
|
||||
if (!user.email) continue;
|
||||
|
||||
const row = usersSheet.addRow([
|
||||
userID,
|
||||
user.emailConfirmed ? 'Yes' : 'No',
|
||||
creationDate,
|
||||
user.lastLogin ? new Date(user.lastLogin) : '',
|
||||
{formula: `=HYPERLINK("#'Orders'!A${firstOrderRow}", "${user.purchases.length} purchase${user.purchases.length > 1 ? "s" : ""}")`},
|
||||
user.isAdmin ? 'Yes' : 'No'
|
||||
])
|
||||
|
||||
row.eachCell(cell => {cell.alignment = {horizontal: 'left' }})
|
||||
}
|
||||
|
||||
ordersSheet.getColumn('itemCost').eachCell(cell => {cell.numFmt = '$0.00'})
|
||||
ordersSheet.getColumn('shippingEst').eachCell(cell => {cell.numFmt = '$0.00'})
|
||||
ordersSheet.getColumn('shippingCost').eachCell(cell => {cell.numFmt = '$0.00'})
|
||||
ordersSheet.getColumn('totalCost').eachCell(cell => {cell.numFmt = '$0.00'})
|
||||
|
||||
couponsSheet.getColumn('flat').eachCell(cell => {cell.numFmt = '$0.00'})
|
||||
couponsSheet.getColumn('percent').eachCell(cell => {cell.numFmt = '0%'})
|
||||
couponsSheet.getColumn('perSock').eachCell(cell => {cell.numFmt = '$0.00'})
|
||||
|
||||
await workbook.xlsx.writeFile(filename);
|
||||
}
|
||||
|
||||
function formatMoney(value) {
|
||||
if(value === undefined) return ''
|
||||
if(value === '') return ''
|
||||
if(value === 0) return ''
|
||||
|
||||
if(typeof value === 'string')
|
||||
return parseFloat(value);
|
||||
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
function formatAddress(address) {
|
||||
if(typeof address !== 'object')
|
||||
return address
|
||||
|
||||
return `${address.name || ""} | ${address.company ? address.company + "| " : ""}${address.street1} | ${address.street2 ? address.street2 + "| " : ""}${address.city} | ${address.state} | ${address.country} | ${address.zip}`
|
||||
}
|
Loading…
Reference in New Issue