1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
'use strict';
const program = require('commander'),
pkg = require('./package.json'),
readline = require('readline'),
request = require("request"),
fs = require('fs');
function extractDataFromSite(siteUrl, outputDir){
Promise.resolve(1).then(function fetchPage(pageNo) {
extractProductsFromPage(siteUrl, pageNo).then(products => {
writeProductsAsMarkdown(products, outputDir).catch(err => console.log(err));
if(products.length > 0) {
fetchPage(pageNo + 1);
}
}).catch(err => console.log(err));
}).catch(err => console.log(err));
}
function getFileContent(product){
return getFrontMatter(product) + "\n\n" + getContent(product);
}
function getFrontMatter(product) {
let frontMatter = {}
frontMatter.title = product.title;
frontMatter.date = new Date().toISOString()
frontMatter.tags = product.tags;
frontMatter.categories = product.product_type;
frontMatter.images = product.images.map(image => image.src);
frontMatter.thumbnailImage = product.images[0].src;
if(product.options[0].name == "Title"){
frontMatter.options = {}
} else {
frontMatter.options = product.options.reduce((map, option) => {
map[option.name] = option.values;
return map;
}, {});
}
frontMatter.actualPrice = null;
frontMatter.comparePrice = null;
frontMatter.inStock = null;
if(product.variants[0].title === "Default Title") {
frontMatter.variants = [];
frontMatter.actualPrice = product.variants[0].price;
if(product.variants[0].compare_at_price && product.variants[0].compare_at_price > 0) {
frontMatter.comparePrice = product.variants[0].compare_at_price;
}
frontMatter.inStock = product.variants[0].available;
} else {
frontMatter.variants = product.variants.map(variant => {
let frontMatterVariant = {};
frontMatterVariant.optionCombination = [variant.option1, variant.option2, variant.option3].filter(val => val != null);
frontMatterVariant.actualPrice = variant.price;
frontMatterVariant.comparePrice = variant.compare_at_price;
frontMatterVariant.inStock = variant.available;
return frontMatterVariant;
});
}
return JSON.stringify(frontMatter, null, 4);
}
function getContent(product){
return product.body_html;
}
function writeProductsAsMarkdown(products, outputDir){
return new Promise(function (resolve, reject){
products.forEach(product => {
let outputFileName = product.title.replace(/\W/g, '-').replace(/-+/g, "-").toLowerCase();
let outputFile = outputDir + "/" + outputFileName + ".md";
fs.writeFile(outputFile, getFileContent(product), function(err) {
if(err) {
reject(err);
}
});
});
});
}
function extractProductsFromPage(siteUrl, pageNo){
let url = siteUrl + "products.json";
let queryParams = {"page": pageNo};
let headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
};
return new Promise(function(resolve, reject){
request.get({url, headers, qs:queryParams}, (err, resp, body) => {
if(err) {
reject(err);
}
let products = JSON.parse(body).products;
resolve(products);
});
});
}
function importProducts(outputDirectory) {
let rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
terminal: false
});
rl.on('line', function (siteUrl) {
extractDataFromSite(siteUrl, outputDirectory);
});
}
program
.version(pkg.version)
.description('Import products from shopify site')
.arguments('[output-directory]')
.action(importProducts)
.parse(process.argv);
if (program.args.length === 0) program.help();
|
