Initial commit

This commit is contained in:
Spencer Pincott
2024-07-15 22:20:13 -04:00
commit 97737ca1ae
16618 changed files with 934131 additions and 0 deletions

16
themes/keepit/node_modules/@algolia/cli/commands.js generated vendored Normal file
View File

@@ -0,0 +1,16 @@
module.exports = {
addrules: require('./commands/AddRules.js'),
addsynonyms: require('./commands/AddSynonyms.js'),
deleteindicespattern: require('./commands/DeleteIndicesPattern.js'),
export: require('./commands/Export.js'),
exportrules: require('./commands/ExportRules.js'),
exportsynonyms: require('./commands/ExportSynonyms.js'),
getsettings: require('./commands/GetSettings.js'),
import: require('./commands/Import.js'),
interactive: require('./commands/Interactive.js'),
search: require('./commands/Search.js'),
setsettings: require('./commands/SetSettings.js'),
transferindex: require('./commands/TransferIndex.js'),
transferindexconfig: require('./commands/TransferIndexConfig.js'),
transformlines: require('./commands/TransformLines.js'),
};

View File

@@ -0,0 +1,72 @@
const fs = require('fs');
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class AddRulesScript extends Base {
constructor() {
super();
// Bind class methods
this.getSource = this.getSource.bind(this);
this.parseBatchRulesOptions = this.parseBatchRulesOptions.bind(this);
this.start = this.start.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia addrules -a algoliaappid -k algoliaapikey -n algoliaindexname -s sourcefilepath -p batchRulesParams\n\n';
this.params = [
'algoliaappid',
'algoliaapikey',
'algoliaindexname',
'sourcefilepath',
];
}
getSource(path) {
const filepath = this.normalizePath(path);
if (!fs.lstatSync(filepath).isFile())
throw new Error('Source filepath must target valid rules file.');
return filepath;
}
parseBatchRulesOptions(params) {
try {
const options = { forwardToReplicas: false, clearExistingRules: false };
if (params === null) return options;
else return JSON.parse(params);
} catch (e) {
throw e;
}
}
async start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const appId = program.algoliaappid;
const apiKey = program.algoliaapikey;
const indexName = program.algoliaindexname;
const sourcefilepath = program.sourcefilepath;
const params = program.params || null;
// Get rules
const rulesPath = this.getSource(sourcefilepath);
const rulesFile = await fs.readFileSync(rulesPath);
const rules = JSON.parse(rulesFile);
// Get options
const batchRulesOptions = this.parseBatchRulesOptions(params);
// Instantiate Algolia index
const client = algolia(appId, apiKey);
const index = client.initIndex(indexName);
// Add rules
const result = await index.batchRules(rules, batchRulesOptions);
return console.log(result);
} catch (e) {
throw e;
}
}
}
const addRulesScript = new AddRulesScript();
module.exports = addRulesScript;

View File

@@ -0,0 +1,89 @@
const fs = require('fs');
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class AddSynonymsScript extends Base {
constructor() {
super();
// Bind class methods
this.getSource = this.getSource.bind(this);
this.parseBatchSynonymsOptions = this.parseBatchSynonymsOptions.bind(this);
this.convertCsvToJson = this.convertCsvToJson.bind(this);
this.start = this.start.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia addsynonyms -a algoliaappid -k algoliaapikey -n algoliaindexname -s sourcefilepath -p batchSynonymsParams\n\n';
this.params = [
'algoliaappid',
'algoliaapikey',
'algoliaindexname',
'sourcefilepath',
];
}
getSource(path) {
const filepath = this.normalizePath(path);
if (!fs.lstatSync(filepath).isFile())
throw new Error('Source filepath must target valid synonyms file.');
return filepath;
}
parseBatchSynonymsOptions(params) {
try {
const options = {
forwardToReplicas: false,
clearExistingSynonyms: false,
};
if (params === null) return options;
else return JSON.parse(params);
} catch (e) {
throw e;
}
}
convertCsvToJson(synonymFile, filepath) {
const synonyms = synonymFile.toString().split('\n');
return synonyms.map((line, num) => ({
type: 'synonym',
objectID: `${filepath}-${num}`,
synonyms: line.split(','),
}));
}
async start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const appId = program.algoliaappid;
const apiKey = program.algoliaapikey;
const indexName = program.algoliaindexname;
const sourcefilepath = program.sourcefilepath;
const params = program.params || null;
const isCsv = sourcefilepath.split('.').pop() === 'csv';
// Get synonyms
const synonymsPath = this.getSource(sourcefilepath);
const synonymsFile = await fs.readFileSync(synonymsPath);
const synonyms = isCsv
? this.convertCsvToJson(synonymsFile, sourcefilepath)
: JSON.parse(synonymsFile);
// Get options
const batchSynonymsOptions = this.parseBatchSynonymsOptions(params);
// Instantiate Algolia index
const client = algolia(appId, apiKey);
const index = client.initIndex(indexName);
// Add rules
const result = await index.batchSynonyms(synonyms, batchSynonymsOptions);
return console.log(result);
} catch (e) {
throw e;
}
}
}
const addSynonymsScript = new AddSynonymsScript();
module.exports = addSynonymsScript;

View File

@@ -0,0 +1,97 @@
const os = require('os');
const fs = require('fs');
const path = require('path');
const readLine = require('readline');
const chalk = require('chalk');
const speedTest = require('speedtest-net');
class Base {
constructor() {
this.maxHeapMb = process.arch.includes('64') ? 1024 : 512;
}
validate(program, message, params) {
let flag = false;
let output = message;
params.forEach(param => {
if (!program[param]) {
output += chalk.red(`Must specify ${param}\n`);
flag = true;
}
});
if (flag) return program.help(h => h + output);
else return { flag, output };
}
writeProgress(message) {
readLine.clearLine(process.stdout, 0);
readLine.cursorTo(process.stdout, 0);
process.stdout.write(message);
}
normalizePath(input) {
// Convert path input param to valid system absolute path
// Path is absolute, originating from system root
if (path.isAbsolute(input)) return input;
// Path is relative to user's home directory
if (input[0] === '~') return path.join(os.homedir(), input.substr(1));
// Path is relative to current directory
return path.resolve(process.cwd(), input);
}
setSource(options) {
// Set source directory and filenames array
// Used to process path inputs that may either be a single file or a directory of files
const source = this.normalizePath(options.sourceFilepath);
if (fs.lstatSync(source).isDirectory()) {
this.directory = source;
this.filenames = fs.readdirSync(source);
} else if (fs.lstatSync(source).isFile()) {
this.directory = path.parse(source).dir;
this.filenames = [path.parse(source).base];
} else {
throw new Error('Invalid sourcefilepath param');
}
}
getMemoryUsage() {
const used = process.memoryUsage().heapUsed / 1024 / 1024;
const usedMb = Math.round(used * 100) / 100;
const percentUsed = Math.floor((usedMb / this.maxHeapMb) * 100);
return { usedMb, percentUsed };
}
getStringSizeMb(string) {
const bytes = Buffer.byteLength(string, 'utf8');
const mb = bytes / 1024 / 1024;
return Math.ceil(mb);
}
getNetworkSpeed() {
return new Promise((resolve, reject) => {
this.writeProgress('Estimating network speed...');
const test = speedTest({ maxTime: 5000 });
let downloadSpeedMb = null;
let uploadSpeedMb = null;
test.on('error', e => {
console.log(chalk.white.bgRed('Speed test error'), chalk.red(e));
reject(e);
});
test.on('downloadspeed', speed => {
downloadSpeedMb = ((speed * 125) / 1000).toFixed(2);
});
test.on('uploadspeed', speed => {
uploadSpeedMb = ((speed * 125) / 1000).toFixed(2);
});
test.on('done', () => {
console.log(
chalk.blue(`\nDownload: ${downloadSpeedMb} MB/s`),
chalk.blue(`\nUpload: ${uploadSpeedMb} MB/s`)
);
resolve(uploadSpeedMb);
});
});
}
}
module.exports = Base;

View File

@@ -0,0 +1,104 @@
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class DeleteIndicesPatternScript extends Base {
constructor() {
super();
// Define validation constants
this.message =
"\nUsage: $ algolia deleteindices -a algoliaappid -k algoliaapikey -r 'regexp for filtering' -x\n\n";
this.params = ['algoliaappid', 'algoliaapikey', 'regexp', 'dryrun'];
}
removeReplicas({ indices, regexp, dryRun }) {
return Promise.all(
indices.map(async ({ name: indexName }) => {
const index = await this.client.initIndex(indexName);
const indexSettings = await index.getSettings();
const replicas = indexSettings.slaves || indexSettings.replicas;
if (replicas !== undefined && replicas.length > 0) {
const newReplicas = replicas.filter(
replicaIndexName => regexp.test(replicaIndexName) === false
);
if (replicas.length !== newReplicas.length) {
if (dryRun === false) {
const { taskID } = await index.setSettings({
[indexSettings.slaves !== undefined
? 'slaves'
: 'replicas']: newReplicas,
});
await index.waitTask(taskID);
} else {
console.log(
`[DRY RUN] Replicas change on index ${indexName}, \n- before: ${replicas.join(
','
)}\n- after: ${newReplicas.join(',')}`
);
}
}
}
return false;
})
);
}
deleteIndices({ indices, regexp, dryRun }) {
let deletedIndices = 0;
return Promise.all(
indices
.filter(({ name: indexName }) => regexp.test(indexName) === true)
.map(async ({ name: indexName }) => {
deletedIndices++;
if (dryRun === false) {
this.writeProgress(`Deleted indices: ${deletedIndices}`);
const index = this.client.initIndex(indexName);
const { taskID } = await this.client.deleteIndex(indexName);
return index.waitTask(taskID);
} else {
console.log(`[DRY RUN] Delete index ${indexName}`);
return false;
}
})
).then(() => {
console.log('');
if (dryRun === false) {
console.log(`${deletedIndices} indices deleted`);
} else {
console.log(`[DRY RUN] ${deletedIndices} indices deleted`);
}
});
}
async deleteIndicesPattern(options) {
this.client = algolia(options.appId, options.apiKey);
const { items: indices } = await this.client.listIndexes();
const regexp = new RegExp(options.regexp);
await this.removeReplicas({ indices, regexp, dryRun: options.dryRun });
await this.deleteIndices({ indices, regexp, dryRun: options.dryRun });
}
start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const options = {
appId: program.algoliaappid,
apiKey: program.algoliaapikey,
regexp: program.regexp,
dryRun: program.dryrun !== undefined ? program.dryrun === 'true' : true,
};
// Delete indices
return this.deleteIndicesPattern(options);
} catch (e) {
throw e;
}
}
}
module.exports = new DeleteIndicesPatternScript();

View File

@@ -0,0 +1,117 @@
const fs = require('fs');
const path = require('path');
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class ExportScript extends Base {
constructor() {
super();
// Bind class methods
this.getOutput = this.getOutput.bind(this);
this.parseParams = this.parseParams.bind(this);
this.writeFile = this.writeFile.bind(this);
this.exportData = this.exportData.bind(this);
this.start = this.start.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia export -a algoliaappid -k algoliaapikey -n algoliaindexname -o outputpath -p params\n\n';
this.params = ['algoliaappid', 'algoliaapikey', 'algoliaindexname'];
}
getOutput(outputPath) {
// If no outputPath is provided, use directory from which command was invoked
const outputDir =
outputPath !== null ? this.normalizePath(outputPath) : process.cwd();
// Ensure outputPath is a directory
if (!fs.lstatSync(outputDir).isDirectory())
throw new Error('Output path must be a directory.');
return outputDir;
}
parseParams(params) {
try {
if (params === null) return { hitsPerPage: 1000 };
return JSON.parse(params);
} catch (e) {
throw e;
}
}
writeFile(hits, options, fileCount) {
const filename = `algolia-index-${options.indexName}-${fileCount}.json`;
const filePath = path.resolve(options.outputPath, filename);
fs.writeFileSync(filePath, JSON.stringify(hits));
return console.log(`\nDone writing ${filename}`);
}
exportData(options) {
return new Promise((resolve, reject) => {
// Instantiate Algolia index
const client = algolia(options.appId, options.apiKey);
const index = client.initIndex(options.indexName);
// Export index
const browse = index.browseAll('', options.params);
let hits = [];
let hitsCount = 0;
let fileCount = 0;
browse.on('result', result => {
// Push 1000 new hits to array
hits = hits.concat(result.hits);
hitsCount += result.hits.length;
this.writeProgress(`Records browsed: ${hitsCount}`);
if (hits.length >= 10000) {
// Write batch of 10,000 records to file
fileCount++;
this.writeFile(hits, options, fileCount);
// Clear array
hits = [];
}
});
browse.on('end', () => {
if (hits.length > 0) {
// Write remaining records to file
fileCount++;
this.writeFile(hits, options, fileCount);
}
return resolve(
`\nDone exporting index.\nSee your data here: ${options.outputPath}`
);
});
browse.on('error', err => reject(err));
});
}
async start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const options = {
appId: program.algoliaappid,
apiKey: program.algoliaapikey,
indexName: program.algoliaindexname,
outputPath: program.outputpath || null,
params: program.params || null,
};
// Configure and validate output path
options.outputPath = this.getOutput(options.outputPath);
// Configure browseAll params
options.params = this.parseParams(options.params);
// Export data
const result = await this.exportData(options);
return console.log(result);
} catch (e) {
throw e;
}
}
}
const exportScript = new ExportScript();
module.exports = exportScript;

View File

@@ -0,0 +1,61 @@
const fs = require('fs');
const path = require('path');
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class ExportRulesScript extends Base {
constructor() {
super();
// Bind class methods
this.getOutputPath = this.getOutputPath.bind(this);
this.start = this.start.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia exportrules -a algoliaappid -k algoliaapikey -n algoliaindexname -o outputpath\n\n';
this.params = ['algoliaappid', 'algoliaapikey', 'algoliaindexname'];
}
getOutputPath(outputpath, indexName) {
const defaultFilename = `${indexName}-rules.json`;
const defaultFilepath = path.resolve(process.cwd(), defaultFilename);
// Process output filepath
const filepath =
outputpath !== null ? this.normalizePath(outputpath) : defaultFilepath;
// Validate filepath targets valid directory
const dir = path.dirname(filepath);
if (!fs.lstatSync(dir).isDirectory()) {
throw new Error(
`Output path must target valid directory. Eg. ${defaultFilepath}`
);
}
return filepath;
}
async start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const appId = program.algoliaappid;
const apiKey = program.algoliaapikey;
const indexName = program.algoliaindexname;
const outputpath = program.outputpath || null;
const filepath = this.getOutputPath(outputpath, indexName);
// Instantiate Algolia index
const client = algolia(appId, apiKey);
const index = client.initIndex(indexName);
// Get index settings
const rules = await index.exportRules();
fs.writeFileSync(filepath, JSON.stringify(rules));
return console.log(`Done writing ${filepath}`);
} catch (e) {
throw e;
}
}
}
const exportRulesScript = new ExportRulesScript();
module.exports = exportRulesScript;

View File

@@ -0,0 +1,61 @@
const fs = require('fs');
const path = require('path');
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class ExportSynonymsScript extends Base {
constructor() {
super();
// Bind class methods
this.getOutputPath = this.getOutputPath.bind(this);
this.start = this.start.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia exportsynonyms -a algoliaappid -k algoliaapikey -n algoliaindexname -o outputpath\n\n';
this.params = ['algoliaappid', 'algoliaapikey', 'algoliaindexname'];
}
getOutputPath(outputpath, indexName) {
const defaultFilename = `${indexName}-synonyms.json`;
const defaultFilepath = path.resolve(process.cwd(), defaultFilename);
// Process output filepath
const filepath =
outputpath !== null ? this.normalizePath(outputpath) : defaultFilepath;
// Validate filepath targets valid directory
const dir = path.dirname(filepath);
if (!fs.lstatSync(dir).isDirectory()) {
throw new Error(
`Output path must target valid directory. Eg. ${defaultFilepath}`
);
}
return filepath;
}
async start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const appId = program.algoliaappid;
const apiKey = program.algoliaapikey;
const indexName = program.algoliaindexname;
const outputpath = program.outputpath || null;
const filepath = this.getOutputPath(outputpath, indexName);
// Instantiate Algolia index
const client = algolia(appId, apiKey);
const index = client.initIndex(indexName);
// Get index settings
const synonyms = await index.exportSynonyms();
fs.writeFileSync(filepath, JSON.stringify(synonyms));
return console.log(`Done writing ${filepath}`);
} catch (e) {
throw e;
}
}
}
const exportSynonymsScript = new ExportSynonymsScript();
module.exports = exportSynonymsScript;

View File

@@ -0,0 +1,38 @@
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class GetSettingsScript extends Base {
constructor() {
super();
// Bind class methods
this.start = this.start.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia getsettings -a algoliaappid -k algoliaapikey -n algoliaindexname\n\n';
this.params = ['algoliaappid', 'algoliaapikey', 'algoliaindexname'];
}
async start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const appId = program.algoliaappid;
const apiKey = program.algoliaapikey;
const indexName = program.algoliaindexname;
// Instantiate Algolia index
const client = algolia(appId, apiKey);
const index = client.initIndex(indexName);
// Get index settings
const settings = await index.getSettings();
return console.log(JSON.stringify(settings));
} catch (e) {
throw e;
}
}
}
const getSettingsScript = new GetSettingsScript();
module.exports = getSettingsScript;

View File

@@ -0,0 +1,379 @@
const fs = require('fs');
const JSONStream = require('JSONStream');
const through = require('through');
const transform = require('stream-transform');
const Batch = require('batch-stream');
const async = require('async');
const csv = require('csvtojson');
const regexParser = require('regex-parser');
const chalk = require('chalk');
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class ImportScript extends Base {
constructor() {
super();
// Bind class methods
this.defaultTransformations = this.defaultTransformations.bind(this);
this.suggestions = this.suggestions.bind(this);
this.checkMemoryUsage = this.checkMemoryUsage.bind(this);
this.handleHighMemoryUsage = this.handleHighMemoryUsage.bind(this);
this.handleExtremeMemoryUsage = this.handleExtremeMemoryUsage.bind(this);
this.setIndex = this.setIndex.bind(this);
this.setTransformations = this.setTransformations.bind(this);
this.setCsvOptions = this.setCsvOptions.bind(this);
this.conditionallyParseCsv = this.conditionallyParseCsv.bind(this);
this.setBatchSize = this.setBatchSize.bind(this);
this.estimateBatchSize = this.estimateBatchSize.bind(this);
this.updateBatchSize = this.updateBatchSize.bind(this);
this.importToAlgolia = this.importToAlgolia.bind(this);
this.retryImport = this.retryImport.bind(this);
this.indexFiles = this.indexFiles.bind(this);
this.start = this.start.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia import -s sourcefilepath -a algoliaappid -k algoliaapikey -n algoliaindexname -b batchsize -t transformationfilepath -m maxconcurrency -p csvtojsonparams\n\n';
this.params = [
'sourcefilepath',
'algoliaappid',
'algoliaapikey',
'algoliaindexname',
];
}
defaultTransformations(data, cb) {
cb(null, data);
}
suggestions() {
let output = `\nConsider reducing <batchSize> (currently ${
this.batchSize
}).`;
if (this.maxConcurrency > 1)
output += `\nConsider reducing <maxConcurrency> (currently ${
this.maxConcurrency
}).`;
return output;
}
checkMemoryUsage() {
// Exit early if high memory usage warning issued too recently
if (this.highMemoryUsage) return false;
// Get memory usage
const { usedMb, percentUsed } = this.getMemoryUsage();
// Handle if heap usage exceeds n% of estimated allocation for node process
if (percentUsed >= 70) this.handleHighMemoryUsage(percentUsed);
if (percentUsed >= 90) this.handleExtremeMemoryUsage(usedMb, percentUsed);
return false;
}
handleHighMemoryUsage(percentUsed) {
const newBatchSize = Math.floor(this.batchSize / 2);
this.updateBatchSize(newBatchSize);
this.writeProgress(
`High memory usage (${percentUsed}%). Reducing batchSize to ${newBatchSize}`
);
}
handleExtremeMemoryUsage(usedMb, percentUsed) {
// Issue warning
const name = `Warning: High memory usage`;
const message = `Memory usage at ${usedMb} MB (${percentUsed}% of heap allocation for this process).`;
// Set class instance flag to debounce future warnings
this.highMemoryUsage = true;
// Output warning
console.log(
chalk.white.bgRed(`\n${name}`),
chalk.red(`\n${message}`),
chalk.red(`${this.suggestions()}`)
);
// Reset flag in 30 seconds
setTimeout(() => {
this.highMemoryUsage = false;
}, 30000);
}
setIndex(options) {
// Set Algolia index
this.client = algolia(options.appId, options.apiKey);
this.index = this.client.initIndex(options.indexName);
}
setTransformations(options) {
try {
// Set JSON record transformations
const transformations = options.transformations
? require(this.normalizePath(options.transformations))
: null;
// Validate transformations function input param
const valid = transformations && typeof transformations === 'function';
// Assign our transformations function using provided custom transformations file if exists
this.formatRecord = valid ? transformations : this.defaultTransformations;
} catch (e) {
throw e;
}
}
setCsvOptions(options) {
try {
this.csvOptions = options.csvToJsonParams
? JSON.parse(options.csvToJsonParams)
: null;
if (!this.csvOptions) return;
const csvToJsonRegexPropertyList = ['includeColumns', 'ignoreColumns'];
csvToJsonRegexPropertyList.forEach(prop => {
if (this.csvOptions.hasOwnProperty(prop)) {
this.csvOptions[prop] = regexParser(this.csvOptions[prop]);
}
});
} catch (e) {
throw e;
}
}
conditionallyParseCsv(isCsv) {
// Return the appropriate writestream for piping depending on filetype
return isCsv
? csv(this.csvOptions) // Convert from CSV to JSON
: through(); // Do nothing
}
async setBatchSize(options) {
try {
// If user provided batchSize, use and exit early
// Otherwise calculate and set optimal batch size
if (options.objectsPerBatch !== null) {
this.batchSize = options.objectsPerBatch;
return;
}
// Test files to estimate optimal batch size
const estimatedBatchSize = await this.estimateBatchSize();
// Test network upload speed
const uploadSpeedMb = await this.getNetworkSpeed();
// Calculate optimal batch size
this.writeProgress('Calculating optimal batch size...');
let batchSize;
// Reconcile batch size with network speed
if (uploadSpeedMb >= this.desiredBatchSizeMb)
batchSize = Math.floor(estimatedBatchSize);
else
batchSize = Math.floor(
(uploadSpeedMb / this.desiredBatchSizeMb) * estimatedBatchSize
);
// Ensure minimum batch size is enforced
batchSize = Math.max(this.minBatchSize, batchSize);
console.log(chalk.blue(`\nOptimal batch size: ${batchSize}`));
// Set batch size
this.batchSize = batchSize;
} catch (e) {
throw e;
}
}
estimateBatchSize() {
// Read file, estimate average record size, estimate batch size
// Return estimated batch size divided by maxConcurrency
return new Promise((resolve, reject) => {
try {
const filename = this.filenames[0];
const file = `${this.directory}/${filename}`;
const isCsv = filename.split('.').pop() === 'csv';
const fileStream = fs.createReadStream(file, {
autoclose: true,
flags: 'r',
});
this.writeProgress(`Estimating data size...`);
const jsonStreamOption = isCsv ? null : '*';
fileStream
.pipe(this.conditionallyParseCsv(isCsv))
.pipe(JSONStream.parse(jsonStreamOption))
.pipe(transform(this.formatRecord))
.pipe(new Batch({ size: 10000 }))
.pipe(
through(data => {
const count = data.length;
const string = JSON.stringify(data);
const batchSizeMb = this.getStringSizeMb(string);
const avgRecordSizeMb = batchSizeMb / count;
const avgRecordSizeKb = Math.ceil(avgRecordSizeMb * 1000);
const roughBatchSize = this.desiredBatchSizeMb / avgRecordSizeMb;
const estimatedBatchSize = Math.floor(
roughBatchSize / this.maxConcurrency
);
console.log(
chalk.blue(`\nAverage record size: ${avgRecordSizeKb} Kb`)
);
fileStream.destroy();
resolve(estimatedBatchSize);
})
);
} catch (e) {
reject(e);
}
});
}
updateBatchSize(newSize) {
this.batchSize = newSize;
}
getBatchStream() {
return new Batch({ size: this.batchSize });
}
async importToAlgolia(data) {
// Method to index batches of records in Algolia
try {
await this.index.addObjects(data);
this.importCount += data.length;
this.writeProgress(`Records indexed: ${this.importCount}`);
} catch (e) {
let message = e.message;
let addendum = e.stack;
if (e.name === 'AlgoliaSearchRequestTimeoutError') {
message = `You may be attempting to import batches too large for the network connection.`;
addendum = this.suggestions();
this.retryImport(data);
}
console.log(
chalk.white.bgRed(`\nImport error: ${e.name}`),
chalk.red(`\n${message}`),
chalk.red(addendum)
);
throw e;
}
}
retryImport(data) {
// Algolia import retry strategy
try {
this.retryCount++;
console.log(`\n(${this.retryCount}) Retrying batch...`);
const importedBatchCount = Math.floor(this.importCount / this.batchSize);
const retryLimit =
this.retryCount > 15 && this.retryCount > importedBatchCount / 2;
if (retryLimit) {
console.log(
chalk.white.bgRed(`\nError: Failure to index data`),
chalk.red(`\nRetry limit reached.`),
chalk.red(this.suggestions())
);
return;
}
// Split data in half
const middle = Math.floor(data.length / 2);
const firstHalf = data.splice(0, middle);
// Reduce batchsize
if (this.batchSize > middle) this.updateBatchSize(middle);
// Push each half of data into import queue
this.queue.push([firstHalf]);
this.queue.push([data]);
} catch (e) {
console.error('Retry error:', e);
throw e;
}
}
indexFiles(filenames) {
// Recursive method that iterates through an array of filenames, opens a read stream for each file
// then pipes the read stream through a series of transformations (parse CSV/JSON objects, transform
// them, batch them, index them in Algolia) while imposing a queue so that only so many
// indexing threads will be run in parallel
if (filenames.length <= 0) {
console.log('\nDone reading files');
return;
}
// Start new file read stream
// Note: filenames is a reference to the mutable class instance variable this.filenames
const filename = filenames.pop();
const file = `${this.directory}/${filename}`;
const isCsv = filename.split('.').pop() === 'csv';
const fileStream = fs.createReadStream(file, {
autoclose: true,
flags: 'r',
});
fileStream.on('data', () => {
if (this.queue.length() >= this.maxConcurrency) {
// If async upload queue is full, pause reading from file stream
fileStream.pause();
}
});
fileStream.on('end', () => {
// File complete, process next file
this.indexFiles(filenames);
});
// Once the async upload queue is drained, resume reading from file stream
this.queue.drain = () => {
fileStream.resume();
};
// Handle parsing, transforming, batching, and indexing JSON and CSV files
console.log(`\nImporting [${filename}]`);
const jsonStreamOption = isCsv ? null : '*';
fileStream
.pipe(this.conditionallyParseCsv(isCsv, filename))
.pipe(JSONStream.parse(jsonStreamOption))
.pipe(transform(this.formatRecord))
.pipe(this.getBatchStream())
.pipe(
through(data => {
this.checkMemoryUsage();
this.queue.push([data]);
})
);
}
async start(program) {
// Script reads JSON or CSV file, or directory of such files, optionally applies
// transformations, then batches and indexes the data in Algolia.
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const options = {
sourceFilepath: program.sourcefilepath,
appId: program.algoliaappid,
apiKey: program.algoliaapikey,
indexName: program.algoliaindexname,
objectsPerBatch: program.batchsize || null,
transformations: program.transformationfilepath || null,
maxConcurrency: program.maxconcurrency || 2,
csvToJsonParams: program.params || null,
};
// Configure Algolia (this.client, this.index)
this.setIndex(options);
// Configure source paths (this.directory, this.filenames)
this.setSource(options);
// Configure transformations (this.formatRecord)
this.setTransformations(options);
// Configure optional csvtojson params (this.csvOptions)
this.setCsvOptions(options);
// Configure data upload parameters
this.maxConcurrency = options.maxConcurrency;
// Theoretically desirable batch size in MB
this.desiredBatchSizeMb = 10;
// Minimum batch size
this.minBatchSize = 100;
// Configure number of records to index per batch (this.batchSize, this.batch)
await this.setBatchSize(options);
// Assign dangerous memory usage flag
this.highMemoryUsage = false;
// Assign import count
this.importCount = 0;
// Assign retry count
this.retryCount = 0;
// Assign async queue
this.queue = async.queue(this.importToAlgolia, this.maxConcurrency);
// Execute import
console.log(chalk.bgGreen.white('Starting import...'));
return this.indexFiles(this.filenames);
}
}
const importScript = new ImportScript();
module.exports = importScript;

View File

@@ -0,0 +1,60 @@
const inquirer = require('inquirer');
class Interactive {
parseCommandNames(commandList, ownName) {
const names = commandList.map(command => command._name);
// Remove current command name and default command
const commandNames = names.filter(name => name !== ownName && name !== '*');
return commandNames;
}
getCommandQuestion(commandNames) {
return {
type: 'list',
name: 'commandChoice',
message: 'Select the command to run',
choices: commandNames,
};
}
getArgumentQuestions(validArguments) {
return validArguments.map(argument => ({
type: argument.description.includes('key') ? 'password' : 'input',
name: argument.long.substring(2),
message: `${argument.long} | ${argument.description}`,
}));
}
async start(program) {
try {
const commands = require('../commands.js');
const ownName = program._name;
const commandList = program.parent.commands;
// Get list of valid commands
const commandNames = this.parseCommandNames(commandList, ownName);
const commandQuestion = this.getCommandQuestion(commandNames);
// Prompt user to select a command
const commandResponse = await inquirer.prompt(commandQuestion);
// Prepare subsequent questions
const selectedCommand = commandList.find(
command => command._name === commandResponse.commandChoice
);
const validArguments = selectedCommand.options;
const argumentQuestions = this.getArgumentQuestions(validArguments);
// Prompt user to input command arguments
const argumentsResponse = await inquirer.prompt(argumentQuestions);
// Pass arguments to program
const argumentsList = Object.keys(argumentsResponse);
argumentsList.forEach(arg => {
if (argumentsResponse[arg] !== '')
program[arg] = argumentsResponse[arg]; // eslint-disable-line no-param-reassign
});
// Execute selected command
commands[selectedCommand._name].start(program);
} catch (e) {
throw e;
}
}
}
module.exports = new Interactive();

View File

@@ -0,0 +1,68 @@
const fs = require('fs');
const path = require('path');
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class SearchScript extends Base {
constructor() {
super();
// Bind class methods
this.start = this.start.bind(this);
this.parseSearchOptions = this.parseSearchOptions.bind(this);
this.writeOutput = this.writeOutput.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia search -a algoliaappid -k algoliaapikey -n algoliaindexname -q query -p searchparams -o outputpath\n\n';
this.params = ['algoliaappid', 'algoliaapikey', 'algoliaindexname'];
}
parseSearchOptions(params) {
return params === null ? {} : JSON.parse(params);
}
async writeOutput(outputFilepath, content) {
const defaultFilepath = path.resolve(process.cwd(), 'search-results.json');
const filepath = this.normalizePath(outputFilepath);
const dir = path.dirname(filepath);
if (!fs.lstatSync(dir).isDirectory()) {
throw new Error(
`Output path must target valid directory. Eg. ${defaultFilepath}`
);
} else {
await fs.writeFileSync(filepath, content);
}
}
async start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const appId = program.algoliaappid;
const apiKey = program.algoliaapikey;
const indexName = program.algoliaindexname;
const query = program.query || '';
const params = program.params || null;
const outputPath = program.outputpath || null;
// Get options
const options = this.parseSearchOptions(params);
// Instantiate Algolia index
const client = algolia(appId, apiKey);
const index = client.initIndex(indexName);
// Get index settings
const result = await index.search(query, options);
const output = JSON.stringify(result);
return outputPath === null
? console.log(output)
: await this.writeOutput(outputPath, output);
} catch (e) {
throw e;
}
}
}
const searchScript = new SearchScript();
module.exports = searchScript;

View File

@@ -0,0 +1,72 @@
const fs = require('fs');
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class SetSettingsScript extends Base {
constructor() {
super();
// Bind class methods
this.getSource = this.getSource.bind(this);
this.parseSetSettingsOptions = this.parseSetSettingsOptions.bind(this);
this.start = this.start.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia setsettings -a algoliaappid -k algoliaapikey -n algoliaindexname -s sourcefilepath -p setsettingsparams\n\n';
this.params = [
'algoliaappid',
'algoliaapikey',
'algoliaindexname',
'sourcefilepath',
];
}
getSource(path) {
const filepath = this.normalizePath(path);
if (!fs.lstatSync(filepath).isFile())
throw new Error('Source filepath must target valid settings file.');
return filepath;
}
parseSetSettingsOptions(params) {
try {
const options = { forwardToReplicas: false };
if (params === null) return options;
else return JSON.parse(params);
} catch (e) {
throw e;
}
}
async start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const appId = program.algoliaappid;
const apiKey = program.algoliaapikey;
const indexName = program.algoliaindexname;
const sourceFilepath = program.sourcefilepath;
const params = program.params || null;
// Get index settings
const settingsPath = this.getSource(sourceFilepath);
const settingsFile = await fs.readFileSync(settingsPath);
const settings = JSON.parse(settingsFile);
// Get options
const settingsOptions = this.parseSetSettingsOptions(params);
// Instantiate Algolia index
const client = algolia(appId, apiKey);
const index = client.initIndex(indexName);
// Set index settings
const result = await index.setSettings(settings, settingsOptions);
return console.log(result);
} catch (e) {
throw e;
}
}
}
const setSettingsScript = new SetSettingsScript();
module.exports = setSettingsScript;

View File

@@ -0,0 +1,126 @@
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class TransferIndexScript extends Base {
constructor() {
super();
// Bind class methods
this.getIndices = this.getIndices.bind(this);
this.getTransformations = this.getTransformations.bind(this);
this.transferIndexConfig = this.transferIndexConfig.bind(this);
this.transferData = this.transferData.bind(this);
this.start = this.start.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia transferindex -a sourcealgoliaappid -k sourcealgoliaapikey -n sourcealgoliaindexname -d destinationalgoliaappid -y destinationalgoliaapikey -i destinationindexname -t transformationfilepath -e true\n\n';
this.params = [
'sourcealgoliaappid',
'sourcealgoliaapikey',
'sourcealgoliaindexname',
'destinationalgoliaappid',
'destinationalgoliaapikey',
];
}
getIndices(options) {
// Instantiate Algolia indices
const sourceClient = algolia(options.sourceAppId, options.sourceApiKey);
const sourceIndex = sourceClient.initIndex(options.sourceIndexName);
const destinationClient = algolia(
options.destinationAppId,
options.destinationApiKey
);
const destinationIndex = destinationClient.initIndex(
options.destinationIndexName
);
return { sourceIndex, destinationIndex };
}
getTransformations(options) {
// Set JSON record transformations
const transformations = options.transformations
? require(options.transformations)
: null;
// Validate transformations function input param
const valid = transformations && typeof transformations === 'function';
// Return provided transformation function if exists
return valid ? transformations : null;
}
async transferIndexConfig(indices, options) {
// Transfer settings, synonyms, and query rules
const settings = await indices.sourceIndex.getSettings();
const synonyms = await indices.sourceIndex.exportSynonyms();
const rules = await indices.sourceIndex.exportRules();
if (options.excludeReplicas) delete settings.replicas;
await indices.destinationIndex.setSettings(settings);
await indices.destinationIndex.batchSynonyms(synonyms);
await indices.destinationIndex.batchRules(rules);
}
transferData(indices, formatRecord) {
return new Promise((resolve, reject) => {
// Export index
const browse = indices.sourceIndex.browseAll('', {
attributesToRetrieve: ['*'],
});
let hitsCount = 0;
// Set browseAll event handlers
browse.on('result', async result => {
// Push hits to destination index
try {
const hits = formatRecord
? result.hits.map(formatRecord)
: result.hits;
await indices.destinationIndex.addObjects(hits);
hitsCount += result.hits.length;
this.writeProgress(`Records transferred: ${hitsCount}`);
} catch (e) {
throw e;
}
});
browse.on('end', () => resolve('\nDone transferring index.\n'));
browse.on('error', err => reject(err));
});
}
async start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const options = {
sourceAppId: program.sourcealgoliaappid,
sourceApiKey: program.sourcealgoliaapikey,
sourceIndexName: program.sourcealgoliaindexname,
destinationAppId: program.destinationalgoliaappid,
destinationApiKey: program.destinationalgoliaapikey,
destinationIndexName:
program.destinationindexname || program.sourcealgoliaindexname,
transformations: program.transformationfilepath || null,
excludeReplicas:
program.excludereplicas !== undefined
? program.excludereplicas === 'true'
: false,
};
// Configure Algolia clients/indices
const indices = this.getIndices(options);
// Configure transformations
const formatRecord = this.getTransformations(options);
// Transfer index configuration
await this.transferIndexConfig(indices, options);
// Transfer data
const result = await this.transferData(indices, formatRecord);
return console.log(result);
} catch (e) {
throw e;
}
}
}
const transferIndexScript = new TransferIndexScript();
module.exports = transferIndexScript;

View File

@@ -0,0 +1,108 @@
const algolia = require('algoliasearch');
const Base = require('./Base.js');
class TransferIndexConfigScript extends Base {
constructor() {
super();
// Bind class methods
this.start = this.start.bind(this);
this.getIndices = this.getIndices.bind(this);
this.getConfigOptions = this.getConfigOptions.bind(this);
this.transferIndexConfig = this.transferIndexConfig.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia transferindexconfig -a sourcealgoliaappid -k sourcealgoliaapikey -n sourcealgoliaindexname -d destinationalgoliaappid -y destinationalgoliaapikey -i destinationindexname -p configParams -e true\n\n';
this.params = [
'sourcealgoliaappid',
'sourcealgoliaapikey',
'sourcealgoliaindexname',
'destinationalgoliaappid',
'destinationalgoliaapikey',
];
}
getIndices(options) {
// Instantiate Algolia indices
const sourceClient = algolia(options.sourceAppId, options.sourceApiKey);
const sourceIndex = sourceClient.initIndex(options.sourceIndexName);
const destinationClient = algolia(
options.destinationAppId,
options.destinationApiKey
);
const destinationIndex = destinationClient.initIndex(
options.destinationIndexName
);
return { sourceIndex, destinationIndex };
}
getConfigOptions(options) {
// Default config
const config = {
sOptions: {},
rOptions: {},
};
// No params provided, exit early
if (!options.configParams) return config;
const params = JSON.parse(options.configParams);
// Set provided batchSynonyms and batchRules options
if (params.batchSynonymsParams)
config.sOptions = Object.assign({}, params.batchSynonymsParams);
if (params.batchRulesParams)
config.rOptions = Object.assign({}, params.batchRulesParams);
return config;
}
async transferIndexConfig(indices, config, options) {
// Transfer settings, synonyms, and query rules
const settings = await indices.sourceIndex.getSettings();
const synonyms = await indices.sourceIndex.exportSynonyms();
const rules = await indices.sourceIndex.exportRules();
if (options.excludeReplicas) delete settings.replicas;
await indices.destinationIndex.setSettings(settings);
await indices.destinationIndex.batchSynonyms(synonyms, config.sOptions);
await indices.destinationIndex.batchRules(rules, config.rOptions);
}
async start(program) {
try {
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
const options = {
sourceAppId: program.sourcealgoliaappid,
sourceApiKey: program.sourcealgoliaapikey,
sourceIndexName: program.sourcealgoliaindexname,
destinationAppId: program.destinationalgoliaappid,
destinationApiKey: program.destinationalgoliaapikey,
destinationIndexName:
program.destinationindexname || program.sourcealgoliaindexname,
configParams: program.params || null,
excludeReplicas:
program.excludereplicas !== undefined
? program.excludereplicas === 'true'
: false,
};
// Configure Algolia clients/indices
const indices = this.getIndices(options);
// Configure batchSynonyms and batchRules options
const config = this.getConfigOptions(options);
// Transfer index configuration
await this.transferIndexConfig(indices, config, options);
return console.log(
'Index settings, synonyms, and query rules transferred successfully.'
);
} catch (e) {
throw e;
}
}
}
const transferIndexConfigScript = new TransferIndexConfigScript();
module.exports = transferIndexConfigScript;

View File

@@ -0,0 +1,136 @@
const fs = require('fs');
const readLine = require('readline');
const Base = require('./Base.js');
class TransformLinesScript extends Base {
constructor() {
super();
// Bind class methods
this.defaultLineTransformation = this.defaultLineTransformation.bind(this);
this.setOutput = this.setOutput.bind(this);
this.setTransformations = this.setTransformations.bind(this);
this.transformFile = this.transformFile.bind(this);
this.init = this.init.bind(this);
this.start = this.start.bind(this);
// Define validation constants
this.message =
'\nExample: $ algolia transformlines -s sourcefilepath -o outputpath -t transformationfilepath \n\n';
this.params = ['sourcefilepath'];
}
defaultLineTransformation(line) {
// Default line transformation method
/* eslint-disable no-control-regex */
const newLine = line.match(/\u001e/, 'i')
? line.replace(/\u001e/, ',')
: line;
return newLine;
/* eslint-enable no-control-regex */
}
setOutput(outputPath) {
this.outputDir =
outputPath !== null ? this.normalizePath(outputPath) : process.cwd();
// Ensure outputpath is a directory
if (!fs.lstatSync(this.outputDir).isDirectory())
throw new Error('Output path must be a directory.');
}
setTransformations(transformationFilepath) {
try {
// Set JSON record transformations
const transformations = transformationFilepath
? require(this.normalizePath(transformationFilepath))
: null;
// Validate transformations function input param
const valid = transformations && typeof transformations === 'function';
// Assign our transformations function using provided custom transformations file if exists
this.lineTransformation = valid
? transformations
: this.defaultLineTransformation;
} catch (e) {
throw e;
}
}
// Method to transform an individual file line-by-line
transformFile(filename) {
return new Promise((resolve, reject) => {
try {
const writeStream = fs.createWriteStream(
`${this.outputDir}/${filename}`
);
let count = 0;
if (this.transformationFilepath === null) {
writeStream.write('['); // Comment this out to prevent injecting opening bracket at start of new output file
}
const lineReader = readLine.createInterface({
input: fs.createReadStream(`${this.directory}/${filename}`),
});
lineReader.on('line', line => {
count++;
const newLine = this.lineTransformation(line);
this.writeProgress(`Line ${count}...`);
writeStream.write(newLine);
});
lineReader.on('close', () => {
console.log('Done writing!');
if (this.transformationFilepath === null) {
writeStream.write(']'); // Comment this out to prevent injecting closing bracket at end of new output file
}
writeStream.end();
resolve(true);
});
} catch (e) {
reject(e);
}
});
}
// Start script
async init(filenames) {
for (const filename of filenames) {
try {
console.log(`Reading: ${this.directory}/${filename}`);
console.log(`Writing to: ${this.outputDir}/${filename}`);
await this.transformFile(filename);
} catch (e) {
console.log(`Error while processing ${filename}`);
throw new Error(e);
}
}
}
start(program) {
// Script reads a file or directory of files synchronously, line-by-line.
// Writes each file synchronously, line-by-line, to an output directory
// while optionally applying a provided transformation function to each line.
// Validate command; if invalid display help text and exit
this.validate(program, this.message, this.params);
// Config params
this.sourceFilepath = program.sourcefilepath;
this.outputpath = program.outputpath || null;
this.transformationFilepath = program.transformationfilepath || null;
// Configure source paths (this.directory, this.filenames)
this.setSource({ sourceFilepath: this.sourceFilepath });
// Configure output path (this.outputDir)
this.setOutput(this.outputpath);
// Configure transformations (this.lineTransformation)
this.setTransformations(this.transformationFilepath);
// Execute line transformations
this.init(this.filenames);
return false;
}
}
const transformLinesScript = new TransformLinesScript();
module.exports = transformLinesScript;

373
themes/keepit/node_modules/@algolia/cli/index.js generated vendored Executable file
View File

@@ -0,0 +1,373 @@
#!/usr/bin/env node
const program = require('commander');
const { version } = require('./package.json');
const chalk = require('chalk');
const commands = require('./commands.js');
// DOCS
const examples = `
Examples:
$ algolia --help
$ algolia --version
$ algolia interactive
$ algolia search -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -q 'example query' -p '{"filters":["category:book"]}' -o ~/Desktop/results.json
$ algolia import -s ~/Desktop/example_data.json -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -b 5000 -t ~/Desktop/example_transformations.js -m 4 -p '{"delimiter":[":"]}'
$ algolia export -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -o ~/Desktop/output_folder/ -p '{"filters":["category:book"]}'
$ algolia getsettings -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME
$ algolia setsettings -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -s ~/Desktop/example_settings.js -p '{"forwardToReplicas":true}'
$ algolia addrules -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -s ~/Desktop/example_rules.json
$ algolia addsynonyms -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -s ~/Desktop/example_synonyms.csv
$ algolia exportrules -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -o ~/Desktop/output_file.json
$ algolia exportsynonyms -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -o ~/Desktop/output_file.json
$ algolia transferindex -a EXAMPLE_SOURCE_APP_ID -k EXAMPLE_SOURCE_API_KEY -n EXAMPLE_SOURCE_INDEX_NAME -d EXAMPLE_DESTINATION_APP_ID -y EXAMPLE_DESTINATION_API_KEY -i EXAMPLE_DESTINATION_INDEX_NAME -t ~/Desktop/example_transformations.js -e true
$ algolia transferindexconfig -a EXAMPLE_SOURCE_APP_ID -k EXAMPLE_SOURCE_API_KEY -n EXAMPLE_SOURCE_INDEX_NAME -d EXAMPLE_DESTINATION_APP_ID -y EXAMPLE_DESTINATION_API_KEY -i EXAMPLE_DESTINATION_INDEX_NAME -p '{"batchSynonymsParams":{"forwardToReplicas":true}}' -e true
$ algolia deleteindicespattern -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -r '^regex' -x true
$ algolia transformlines -s ~/Desktop/example_source.json -o ~/Desktop/example_output.json -t ~/Desktop/example_transformations.js
$ algolia examples
`;
// HELPERS
const registerDefaultProcessEventListeners = () => {
// Handle process cancellation
process.on('SIGINT', () => {
console.log(chalk.white.bgYellow('\nCancelled'));
process.exit(1);
});
// Handle uncaught exceptions
process.on('uncaughtException', e => {
process.exitCode = 1;
console.log(chalk.white.bgRed('\nUncaught Exception'), chalk.red(`\n${e}`));
});
};
const defaultCommand = command => {
console.error(`Unknown command "${command}".`);
console.error('Run "algolia --help" to view options.');
process.exit(1);
};
// COMMANDS
program.version(version, '-v, --version');
// Search
program
.command('search')
.alias('s')
.description('Search an Algolia index')
.option('-a, --algoliaappid <algoliaAppId>', 'Required | Algolia app ID')
.option('-k, --algoliaapikey <algoliaApiKey>', 'Required | Algolia API key')
.option(
'-n, --algoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.option('-q, --query <query>', 'Optional | Algolia search query string')
.option('-p, --params <params>', 'Optional | Algolia search params')
.option('-o, --outputpath <outputPath>', 'Optional | Output filepath')
.action(cmd => {
commands.search.start(cmd);
});
// Import
program
.command('import')
.alias('i')
.description('Import local JSON or CSV data to an Algolia index')
.option('-s, --sourcefilepath <sourceFilepath>', 'Required | Source filepath')
.option('-a, --algoliaappid <algoliaAppId>', 'Required | Algolia app ID')
.option('-k, --algoliaapikey <algoliaApiKey>', 'Required | Algolia API key')
.option(
'-n, --algoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.option(
'-b, --batchsize <batchSize>',
'Optional | Number of objects to import per batch'
)
.option(
'-t, --transformationfilepath <transformationFilepath>',
'Optional | Transformation filepath'
)
.option(
'-m, --maxconcurrency <maxConcurrency>',
'Optional | Maximum number of concurrent filestreams to process'
)
.option('-p, --params <params>', 'Optional | CsvToJson params')
.action(cmd => {
commands.import.start(cmd);
});
// Export
program
.command('export')
.alias('e')
.description('Export the contents of an Algolia index to local JSON files')
.option('-a, --algoliaappid <algoliaAppId>', 'Required | Algolia app ID')
.option('-k, --algoliaapikey <algoliaApiKey>', 'Required | Algolia API key')
.option(
'-n, --algoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.option('-o, --outputpath <outputPath>', 'Optional | Output filepath')
.option('-p, --params <params>', 'Optional | Algolia browseAll params')
.action(cmd => {
commands.export.start(cmd);
});
// Get Settings
program
.command('getsettings')
.alias('gs')
.description('Get the settings of an Algolia index as JSON')
.option('-a, --algoliaappid <algoliaAppId>', 'Required | Algolia app ID')
.option('-k, --algoliaapikey <algoliaApiKey>', 'Required | Algolia API key')
.option(
'-n, --algoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.action(cmd => {
commands.getsettings.start(cmd);
});
// Set Settings
program
.command('setsettings')
.alias('ss')
.description('Set the settings of an Algolia index from a JSON file')
.option('-a, --algoliaappid <algoliaAppId>', 'Required | Algolia app ID')
.option('-k, --algoliaapikey <algoliaApiKey>', 'Required | Algolia API key')
.option(
'-n, --algoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.option('-s, --sourcefilepath <sourceFilepath>', 'Required | Source filepath')
.option('-p, --params <params>', 'Optional | Algolia setSettings params')
.action(cmd => {
commands.setsettings.start(cmd);
});
// Add Rules
program
.command('addrules')
.alias('ar')
.description('Add query rules to an Algolia index from a JSON file')
.option('-a, --algoliaappid <algoliaAppId>', 'Required | Algolia app ID')
.option('-k, --algoliaapikey <algoliaApiKey>', 'Required | Algolia API key')
.option(
'-n, --algoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.option('-s, --sourcefilepath <sourceFilepath>', 'Required | Source filepath')
.option('-p, --params <params>', 'Optional | Algolia batchRules params')
.action(cmd => {
commands.addrules.start(cmd);
});
// Add Synonyms
program
.command('addsynonyms')
.alias('as')
.description('Add synonyms to an Algolia index from a CSV or JSON file')
.option('-a, --algoliaappid <algoliaAppId>', 'Required | Algolia app ID')
.option('-k, --algoliaapikey <algoliaApiKey>', 'Required | Algolia API key')
.option(
'-n, --algoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.option('-s, --sourcefilepath <sourceFilepath>', 'Required | Source filepath')
.option('-p, --params <params>', 'Optional | Algolia batchSynonyms params')
.action(cmd => {
commands.addsynonyms.start(cmd);
});
// Export Rules
program
.command('exportrules')
.alias('er')
.description('Export the query rules of an Algolia index to local JSON file')
.option('-a, --algoliaappid <algoliaAppId>', 'Required | Algolia app ID')
.option('-k, --algoliaapikey <algoliaApiKey>', 'Required | Algolia API key')
.option(
'-n, --algoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.option('-o, --outputpath <outputPath>', 'Optional | Output filepath')
.action(cmd => {
commands.exportrules.start(cmd);
});
// Export Synonyms
program
.command('exportsynonyms')
.alias('es')
.description('Export the synonyms of an Algolia index to local JSON file')
.option('-a, --algoliaappid <algoliaAppId>', 'Required | Algolia app ID')
.option('-k, --algoliaapikey <algoliaApiKey>', 'Required | Algolia API key')
.option(
'-n, --algoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.option('-o, --outputpath <outputPath>', 'Optional | Output filepath')
.action(cmd => {
commands.exportsynonyms.start(cmd);
});
// Transfer Index
program
.command('transferindex')
.alias('ti')
.description(
'Duplicate the data and settings of an index from one Algolia App to another'
)
.option(
'-a, --sourcealgoliaappid <algoliaAppId>',
'Required | Algolia app ID'
)
.option(
'-k, --sourcealgoliaapikey <algoliaApiKey>',
'Required | Algolia API key'
)
.option(
'-n, --sourcealgoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.option(
'-d, --destinationalgoliaappid <algoliaAppId>',
'Required | Algolia app ID'
)
.option(
'-y, --destinationalgoliaapikey <algoliaApiKey>',
'Required | Algolia API key'
)
.option(
'-i, --destinationindexname <algoliaIndexName>',
'Optional | Algolia index name'
)
.option(
'-t, --transformationfilepath <transformationFilepath>',
'Optional | Transformation filepath'
)
.option(
'-e, --excludereplicas <boolean>',
'Optional | Exclude replicas property of settings object'
)
.action(cmd => {
commands.transferindex.start(cmd);
});
// Transfer Index Config
program
.command('transferindexconfig')
.alias('tig')
.description(
'Duplicate the settings, synonyms, and query rules of an index from one Algolia App to another'
)
.option(
'-a, --sourcealgoliaappid <algoliaAppId>',
'Required | Algolia app ID'
)
.option(
'-k, --sourcealgoliaapikey <algoliaApiKey>',
'Required | Algolia API key'
)
.option(
'-n, --sourcealgoliaindexname <algoliaIndexName>',
'Required | Algolia index name'
)
.option(
'-d, --destinationalgoliaappid <algoliaAppId>',
'Required | Algolia app ID'
)
.option(
'-y, --destinationalgoliaapikey <algoliaApiKey>',
'Required | Algolia API key'
)
.option(
'-i, --destinationindexname <algoliaIndexName>',
'Optional | Algolia index name'
)
.option(
'-p, --params <params>',
'Optional | Algolia batchSynonyms and batchRules params'
)
.option(
'-e, --excludereplicas <boolean>',
'Optional | Exclude replicas property of settings object'
)
.action(cmd => {
commands.transferindexconfig.start(cmd);
});
// Delete Indices
program
.command('deleteindicespattern')
.alias('dip')
.description('Delete multiple indices using a regular expression')
.option('-a, --algoliaappid <algoliaAppId>', 'Required | Algolia app ID')
.option('-k, --algoliaapikey <algoliaApiKey>', 'Required | Algolia API key')
.option('-r, --regexp <regexp>', 'Required | Regexp to use for filtering')
.option(
'-x, --dryrun <boolean>',
'Required | Dry run, will only output what would be done'
)
.action(cmd => {
commands.deleteindicespattern.start(cmd);
});
// Transform Lines
program
.command('transformlines')
.alias('tl')
.description(
'Apply a custom transformation to each line of a file saving output lines to a new file'
)
.option('-s, --sourcefilepath <sourceFilepath>', 'Required | Source filepath')
.option('-o, --outputpath <outputPath>', 'Optional | Output filepath')
.option(
'-t, --transformationfilepath <transformationFilepath>',
'Optional | Transformation filepath'
)
.action(cmd => {
commands.transformlines.start(cmd);
});
// Interactive command
program
.command('interactive')
.alias('shell')
.description('Run in an interactive mode')
.action(cmd => {
commands.interactive.start(cmd);
});
// Display command examples
program
.command('examples')
.alias('ex')
.description('View command examples')
.action(() => {
console.log(examples);
});
// Default Command
program
.command('*')
.alias('default')
.description('Default command if none input')
.action(cmd => {
defaultCommand(cmd);
});
// LOGIC
// Process command
program.parse(process.argv);
// Register node process event listeners
registerDefaultProcessEventListeners();
// Handle no-command case
if (program.args.length === 0) program.help();

View File

@@ -0,0 +1,419 @@
2.20.3 / 2019-10-11
==================
* Support Node.js 0.10 (Revert #1059)
* Ran "npm unpublish commander@2.20.2". There is no 2.20.2.
2.20.1 / 2019-09-29
==================
* Improve executable subcommand tracking
* Update dev dependencies
2.20.0 / 2019-04-02
==================
* fix: resolve symbolic links completely when hunting for subcommands (#935)
* Update index.d.ts (#930)
* Update Readme.md (#924)
* Remove --save option as it isn't required anymore (#918)
* Add link to the license file (#900)
* Added example of receiving args from options (#858)
* Added missing semicolon (#882)
* Add extension to .eslintrc (#876)
2.19.0 / 2018-10-02
==================
* Removed newline after Options and Commands headers (#864)
* Bugfix - Error output (#862)
* Fix to change default value to string (#856)
2.18.0 / 2018-09-07
==================
* Standardize help output (#853)
* chmod 644 travis.yml (#851)
* add support for execute typescript subcommand via ts-node (#849)
2.17.1 / 2018-08-07
==================
* Fix bug in command emit (#844)
2.17.0 / 2018-08-03
==================
* fixed newline output after help information (#833)
* Fix to emit the action even without command (#778)
* npm update (#823)
2.16.0 / 2018-06-29
==================
* Remove Makefile and `test/run` (#821)
* Make 'npm test' run on Windows (#820)
* Add badge to display install size (#807)
* chore: cache node_modules (#814)
* chore: remove Node.js 4 (EOL), add Node.js 10 (#813)
* fixed typo in readme (#812)
* Fix types (#804)
* Update eslint to resolve vulnerabilities in lodash (#799)
* updated readme with custom event listeners. (#791)
* fix tests (#794)
2.15.0 / 2018-03-07
==================
* Update downloads badge to point to graph of downloads over time instead of duplicating link to npm
* Arguments description
2.14.1 / 2018-02-07
==================
* Fix typing of help function
2.14.0 / 2018-02-05
==================
* only register the option:version event once
* Fixes issue #727: Passing empty string for option on command is set to undefined
* enable eqeqeq rule
* resolves #754 add linter configuration to project
* resolves #560 respect custom name for version option
* document how to override the version flag
* document using options per command
2.13.0 / 2018-01-09
==================
* Do not print default for --no-
* remove trailing spaces in command help
* Update CI's Node.js to LTS and latest version
* typedefs: Command and Option types added to commander namespace
2.12.2 / 2017-11-28
==================
* fix: typings are not shipped
2.12.1 / 2017-11-23
==================
* Move @types/node to dev dependency
2.12.0 / 2017-11-22
==================
* add attributeName() method to Option objects
* Documentation updated for options with --no prefix
* typings: `outputHelp` takes a string as the first parameter
* typings: use overloads
* feat(typings): update to match js api
* Print default value in option help
* Fix translation error
* Fail when using same command and alias (#491)
* feat(typings): add help callback
* fix bug when description is add after command with options (#662)
* Format js code
* Rename History.md to CHANGELOG.md (#668)
* feat(typings): add typings to support TypeScript (#646)
* use current node
2.11.0 / 2017-07-03
==================
* Fix help section order and padding (#652)
* feature: support for signals to subcommands (#632)
* Fixed #37, --help should not display first (#447)
* Fix translation errors. (#570)
* Add package-lock.json
* Remove engines
* Upgrade package version
* Prefix events to prevent conflicts between commands and options (#494)
* Removing dependency on graceful-readlink
* Support setting name in #name function and make it chainable
* Add .vscode directory to .gitignore (Visual Studio Code metadata)
* Updated link to ruby commander in readme files
2.10.0 / 2017-06-19
==================
* Update .travis.yml. drop support for older node.js versions.
* Fix require arguments in README.md
* On SemVer you do not start from 0.0.1
* Add missing semi colon in readme
* Add save param to npm install
* node v6 travis test
* Update Readme_zh-CN.md
* Allow literal '--' to be passed-through as an argument
* Test subcommand alias help
* link build badge to master branch
* Support the alias of Git style sub-command
* added keyword commander for better search result on npm
* Fix Sub-Subcommands
* test node.js stable
* Fixes TypeError when a command has an option called `--description`
* Update README.md to make it beginner friendly and elaborate on the difference between angled and square brackets.
* Add chinese Readme file
2.9.0 / 2015-10-13
==================
* Add option `isDefault` to set default subcommand #415 @Qix-
* Add callback to allow filtering or post-processing of help text #434 @djulien
* Fix `undefined` text in help information close #414 #416 @zhiyelee
2.8.1 / 2015-04-22
==================
* Back out `support multiline description` Close #396 #397
2.8.0 / 2015-04-07
==================
* Add `process.execArg` support, execution args like `--harmony` will be passed to sub-commands #387 @DigitalIO @zhiyelee
* Fix bug in Git-style sub-commands #372 @zhiyelee
* Allow commands to be hidden from help #383 @tonylukasavage
* When git-style sub-commands are in use, yet none are called, display help #382 @claylo
* Add ability to specify arguments syntax for top-level command #258 @rrthomas
* Support multiline descriptions #208 @zxqfox
2.7.1 / 2015-03-11
==================
* Revert #347 (fix collisions when option and first arg have same name) which causes a bug in #367.
2.7.0 / 2015-03-09
==================
* Fix git-style bug when installed globally. Close #335 #349 @zhiyelee
* Fix collisions when option and first arg have same name. Close #346 #347 @tonylukasavage
* Add support for camelCase on `opts()`. Close #353 @nkzawa
* Add node.js 0.12 and io.js to travis.yml
* Allow RegEx options. #337 @palanik
* Fixes exit code when sub-command failing. Close #260 #332 @pirelenito
* git-style `bin` files in $PATH make sense. Close #196 #327 @zhiyelee
2.6.0 / 2014-12-30
==================
* added `Command#allowUnknownOption` method. Close #138 #318 @doozr @zhiyelee
* Add application description to the help msg. Close #112 @dalssoft
2.5.1 / 2014-12-15
==================
* fixed two bugs incurred by variadic arguments. Close #291 @Quentin01 #302 @zhiyelee
2.5.0 / 2014-10-24
==================
* add support for variadic arguments. Closes #277 @whitlockjc
2.4.0 / 2014-10-17
==================
* fixed a bug on executing the coercion function of subcommands option. Closes #270
* added `Command.prototype.name` to retrieve command name. Closes #264 #266 @tonylukasavage
* added `Command.prototype.opts` to retrieve all the options as a simple object of key-value pairs. Closes #262 @tonylukasavage
* fixed a bug on subcommand name. Closes #248 @jonathandelgado
* fixed function normalize doesnt honor option terminator. Closes #216 @abbr
2.3.0 / 2014-07-16
==================
* add command alias'. Closes PR #210
* fix: Typos. Closes #99
* fix: Unused fs module. Closes #217
2.2.0 / 2014-03-29
==================
* add passing of previous option value
* fix: support subcommands on windows. Closes #142
* Now the defaultValue passed as the second argument of the coercion function.
2.1.0 / 2013-11-21
==================
* add: allow cflag style option params, unit test, fixes #174
2.0.0 / 2013-07-18
==================
* remove input methods (.prompt, .confirm, etc)
1.3.2 / 2013-07-18
==================
* add support for sub-commands to co-exist with the original command
1.3.1 / 2013-07-18
==================
* add quick .runningCommand hack so you can opt-out of other logic when running a sub command
1.3.0 / 2013-07-09
==================
* add EACCES error handling
* fix sub-command --help
1.2.0 / 2013-06-13
==================
* allow "-" hyphen as an option argument
* support for RegExp coercion
1.1.1 / 2012-11-20
==================
* add more sub-command padding
* fix .usage() when args are present. Closes #106
1.1.0 / 2012-11-16
==================
* add git-style executable subcommand support. Closes #94
1.0.5 / 2012-10-09
==================
* fix `--name` clobbering. Closes #92
* fix examples/help. Closes #89
1.0.4 / 2012-09-03
==================
* add `outputHelp()` method.
1.0.3 / 2012-08-30
==================
* remove invalid .version() defaulting
1.0.2 / 2012-08-24
==================
* add `--foo=bar` support [arv]
* fix password on node 0.8.8. Make backward compatible with 0.6 [focusaurus]
1.0.1 / 2012-08-03
==================
* fix issue #56
* fix tty.setRawMode(mode) was moved to tty.ReadStream#setRawMode() (i.e. process.stdin.setRawMode())
1.0.0 / 2012-07-05
==================
* add support for optional option descriptions
* add defaulting of `.version()` to package.json's version
0.6.1 / 2012-06-01
==================
* Added: append (yes or no) on confirmation
* Added: allow node.js v0.7.x
0.6.0 / 2012-04-10
==================
* Added `.prompt(obj, callback)` support. Closes #49
* Added default support to .choose(). Closes #41
* Fixed the choice example
0.5.1 / 2011-12-20
==================
* Fixed `password()` for recent nodes. Closes #36
0.5.0 / 2011-12-04
==================
* Added sub-command option support [itay]
0.4.3 / 2011-12-04
==================
* Fixed custom help ordering. Closes #32
0.4.2 / 2011-11-24
==================
* Added travis support
* Fixed: line-buffered input automatically trimmed. Closes #31
0.4.1 / 2011-11-18
==================
* Removed listening for "close" on --help
0.4.0 / 2011-11-15
==================
* Added support for `--`. Closes #24
0.3.3 / 2011-11-14
==================
* Fixed: wait for close event when writing help info [Jerry Hamlet]
0.3.2 / 2011-11-01
==================
* Fixed long flag definitions with values [felixge]
0.3.1 / 2011-10-31
==================
* Changed `--version` short flag to `-V` from `-v`
* Changed `.version()` so it's configurable [felixge]
0.3.0 / 2011-10-31
==================
* Added support for long flags only. Closes #18
0.2.1 / 2011-10-24
==================
* "node": ">= 0.4.x < 0.7.0". Closes #20
0.2.0 / 2011-09-26
==================
* Allow for defaults that are not just boolean. Default peassignment only occurs for --no-*, optional, and required arguments. [Jim Isaacs]
0.1.0 / 2011-08-24
==================
* Added support for custom `--help` output
0.0.5 / 2011-08-18
==================
* Changed: when the user enters nothing prompt for password again
* Fixed issue with passwords beginning with numbers [NuckChorris]
0.0.4 / 2011-08-15
==================
* Fixed `Commander#args`
0.0.3 / 2011-08-15
==================
* Added default option value support
0.0.2 / 2011-08-15
==================
* Added mask support to `Command#password(str[, mask], fn)`
* Added `Command#password(str, fn)`
0.0.1 / 2010-01-03
==================
* Initial release

View File

@@ -0,0 +1,22 @@
(The MIT License)
Copyright (c) 2011 TJ Holowaychuk <tj@vision-media.ca>
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@@ -0,0 +1,428 @@
# Commander.js
[![Build Status](https://api.travis-ci.org/tj/commander.js.svg?branch=master)](http://travis-ci.org/tj/commander.js)
[![NPM Version](http://img.shields.io/npm/v/commander.svg?style=flat)](https://www.npmjs.org/package/commander)
[![NPM Downloads](https://img.shields.io/npm/dm/commander.svg?style=flat)](https://npmcharts.com/compare/commander?minimal=true)
[![Install Size](https://packagephobia.now.sh/badge?p=commander)](https://packagephobia.now.sh/result?p=commander)
[![Join the chat at https://gitter.im/tj/commander.js](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/tj/commander.js?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
The complete solution for [node.js](http://nodejs.org) command-line interfaces, inspired by Ruby's [commander](https://github.com/commander-rb/commander).
[API documentation](http://tj.github.com/commander.js/)
## Installation
$ npm install commander
## Option parsing
Options with commander are defined with the `.option()` method, also serving as documentation for the options. The example below parses args and options from `process.argv`, leaving remaining args as the `program.args` array which were not consumed by options.
```js
#!/usr/bin/env node
/**
* Module dependencies.
*/
var program = require('commander');
program
.version('0.1.0')
.option('-p, --peppers', 'Add peppers')
.option('-P, --pineapple', 'Add pineapple')
.option('-b, --bbq-sauce', 'Add bbq sauce')
.option('-c, --cheese [type]', 'Add the specified type of cheese [marble]', 'marble')
.parse(process.argv);
console.log('you ordered a pizza with:');
if (program.peppers) console.log(' - peppers');
if (program.pineapple) console.log(' - pineapple');
if (program.bbqSauce) console.log(' - bbq');
console.log(' - %s cheese', program.cheese);
```
Short flags may be passed as a single arg, for example `-abc` is equivalent to `-a -b -c`. Multi-word options such as "--template-engine" are camel-cased, becoming `program.templateEngine` etc.
Note that multi-word options starting with `--no` prefix negate the boolean value of the following word. For example, `--no-sauce` sets the value of `program.sauce` to false.
```js
#!/usr/bin/env node
/**
* Module dependencies.
*/
var program = require('commander');
program
.option('--no-sauce', 'Remove sauce')
.parse(process.argv);
console.log('you ordered a pizza');
if (program.sauce) console.log(' with sauce');
else console.log(' without sauce');
```
To get string arguments from options you will need to use angle brackets <> for required inputs or square brackets [] for optional inputs.
e.g. ```.option('-m --myarg [myVar]', 'my super cool description')```
Then to access the input if it was passed in.
e.g. ```var myInput = program.myarg```
**NOTE**: If you pass a argument without using brackets the example above will return true and not the value passed in.
## Version option
Calling the `version` implicitly adds the `-V` and `--version` options to the command.
When either of these options is present, the command prints the version number and exits.
$ ./examples/pizza -V
0.0.1
If you want your program to respond to the `-v` option instead of the `-V` option, simply pass custom flags to the `version` method using the same syntax as the `option` method.
```js
program
.version('0.0.1', '-v, --version')
```
The version flags can be named anything, but the long option is required.
## Command-specific options
You can attach options to a command.
```js
#!/usr/bin/env node
var program = require('commander');
program
.command('rm <dir>')
.option('-r, --recursive', 'Remove recursively')
.action(function (dir, cmd) {
console.log('remove ' + dir + (cmd.recursive ? ' recursively' : ''))
})
program.parse(process.argv)
```
A command's options are validated when the command is used. Any unknown options will be reported as an error. However, if an action-based command does not define an action, then the options are not validated.
## Coercion
```js
function range(val) {
return val.split('..').map(Number);
}
function list(val) {
return val.split(',');
}
function collect(val, memo) {
memo.push(val);
return memo;
}
function increaseVerbosity(v, total) {
return total + 1;
}
program
.version('0.1.0')
.usage('[options] <file ...>')
.option('-i, --integer <n>', 'An integer argument', parseInt)
.option('-f, --float <n>', 'A float argument', parseFloat)
.option('-r, --range <a>..<b>', 'A range', range)
.option('-l, --list <items>', 'A list', list)
.option('-o, --optional [value]', 'An optional value')
.option('-c, --collect [value]', 'A repeatable value', collect, [])
.option('-v, --verbose', 'A value that can be increased', increaseVerbosity, 0)
.parse(process.argv);
console.log(' int: %j', program.integer);
console.log(' float: %j', program.float);
console.log(' optional: %j', program.optional);
program.range = program.range || [];
console.log(' range: %j..%j', program.range[0], program.range[1]);
console.log(' list: %j', program.list);
console.log(' collect: %j', program.collect);
console.log(' verbosity: %j', program.verbose);
console.log(' args: %j', program.args);
```
## Regular Expression
```js
program
.version('0.1.0')
.option('-s --size <size>', 'Pizza size', /^(large|medium|small)$/i, 'medium')
.option('-d --drink [drink]', 'Drink', /^(coke|pepsi|izze)$/i)
.parse(process.argv);
console.log(' size: %j', program.size);
console.log(' drink: %j', program.drink);
```
## Variadic arguments
The last argument of a command can be variadic, and only the last argument. To make an argument variadic you have to
append `...` to the argument name. Here is an example:
```js
#!/usr/bin/env node
/**
* Module dependencies.
*/
var program = require('commander');
program
.version('0.1.0')
.command('rmdir <dir> [otherDirs...]')
.action(function (dir, otherDirs) {
console.log('rmdir %s', dir);
if (otherDirs) {
otherDirs.forEach(function (oDir) {
console.log('rmdir %s', oDir);
});
}
});
program.parse(process.argv);
```
An `Array` is used for the value of a variadic argument. This applies to `program.args` as well as the argument passed
to your action as demonstrated above.
## Specify the argument syntax
```js
#!/usr/bin/env node
var program = require('commander');
program
.version('0.1.0')
.arguments('<cmd> [env]')
.action(function (cmd, env) {
cmdValue = cmd;
envValue = env;
});
program.parse(process.argv);
if (typeof cmdValue === 'undefined') {
console.error('no command given!');
process.exit(1);
}
console.log('command:', cmdValue);
console.log('environment:', envValue || "no environment given");
```
Angled brackets (e.g. `<cmd>`) indicate required input. Square brackets (e.g. `[env]`) indicate optional input.
## Git-style sub-commands
```js
// file: ./examples/pm
var program = require('commander');
program
.version('0.1.0')
.command('install [name]', 'install one or more packages')
.command('search [query]', 'search with optional query')
.command('list', 'list packages installed', {isDefault: true})
.parse(process.argv);
```
When `.command()` is invoked with a description argument, no `.action(callback)` should be called to handle sub-commands, otherwise there will be an error. This tells commander that you're going to use separate executables for sub-commands, much like `git(1)` and other popular tools.
The commander will try to search the executables in the directory of the entry script (like `./examples/pm`) with the name `program-command`, like `pm-install`, `pm-search`.
Options can be passed with the call to `.command()`. Specifying `true` for `opts.noHelp` will remove the subcommand from the generated help output. Specifying `true` for `opts.isDefault` will run the subcommand if no other subcommand is specified.
If the program is designed to be installed globally, make sure the executables have proper modes, like `755`.
### `--harmony`
You can enable `--harmony` option in two ways:
* Use `#! /usr/bin/env node --harmony` in the sub-commands scripts. Note some os version dont support this pattern.
* Use the `--harmony` option when call the command, like `node --harmony examples/pm publish`. The `--harmony` option will be preserved when spawning sub-command process.
## Automated --help
The help information is auto-generated based on the information commander already knows about your program, so the following `--help` info is for free:
```
$ ./examples/pizza --help
Usage: pizza [options]
An application for pizzas ordering
Options:
-h, --help output usage information
-V, --version output the version number
-p, --peppers Add peppers
-P, --pineapple Add pineapple
-b, --bbq Add bbq sauce
-c, --cheese <type> Add the specified type of cheese [marble]
-C, --no-cheese You do not want any cheese
```
## Custom help
You can display arbitrary `-h, --help` information
by listening for "--help". Commander will automatically
exit once you are done so that the remainder of your program
does not execute causing undesired behaviors, for example
in the following executable "stuff" will not output when
`--help` is used.
```js
#!/usr/bin/env node
/**
* Module dependencies.
*/
var program = require('commander');
program
.version('0.1.0')
.option('-f, --foo', 'enable some foo')
.option('-b, --bar', 'enable some bar')
.option('-B, --baz', 'enable some baz');
// must be before .parse() since
// node's emit() is immediate
program.on('--help', function(){
console.log('')
console.log('Examples:');
console.log(' $ custom-help --help');
console.log(' $ custom-help -h');
});
program.parse(process.argv);
console.log('stuff');
```
Yields the following help output when `node script-name.js -h` or `node script-name.js --help` are run:
```
Usage: custom-help [options]
Options:
-h, --help output usage information
-V, --version output the version number
-f, --foo enable some foo
-b, --bar enable some bar
-B, --baz enable some baz
Examples:
$ custom-help --help
$ custom-help -h
```
## .outputHelp(cb)
Output help information without exiting.
Optional callback cb allows post-processing of help text before it is displayed.
If you want to display help by default (e.g. if no command was provided), you can use something like:
```js
var program = require('commander');
var colors = require('colors');
program
.version('0.1.0')
.command('getstream [url]', 'get stream URL')
.parse(process.argv);
if (!process.argv.slice(2).length) {
program.outputHelp(make_red);
}
function make_red(txt) {
return colors.red(txt); //display the help text in red on the console
}
```
## .help(cb)
Output help information and exit immediately.
Optional callback cb allows post-processing of help text before it is displayed.
## Custom event listeners
You can execute custom actions by listening to command and option events.
```js
program.on('option:verbose', function () {
process.env.VERBOSE = this.verbose;
});
// error on unknown commands
program.on('command:*', function () {
console.error('Invalid command: %s\nSee --help for a list of available commands.', program.args.join(' '));
process.exit(1);
});
```
## Examples
```js
var program = require('commander');
program
.version('0.1.0')
.option('-C, --chdir <path>', 'change the working directory')
.option('-c, --config <path>', 'set config path. defaults to ./deploy.conf')
.option('-T, --no-tests', 'ignore test hook');
program
.command('setup [env]')
.description('run setup commands for all envs')
.option("-s, --setup_mode [mode]", "Which setup mode to use")
.action(function(env, options){
var mode = options.setup_mode || "normal";
env = env || 'all';
console.log('setup for %s env(s) with %s mode', env, mode);
});
program
.command('exec <cmd>')
.alias('ex')
.description('execute the given remote cmd')
.option("-e, --exec_mode <mode>", "Which exec mode to use")
.action(function(cmd, options){
console.log('exec "%s" using %s mode', cmd, options.exec_mode);
}).on('--help', function() {
console.log('');
console.log('Examples:');
console.log('');
console.log(' $ deploy exec sequential');
console.log(' $ deploy exec async');
});
program
.command('*')
.action(function(env){
console.log('deploying "%s"', env);
});
program.parse(process.argv);
```
More Demos can be found in the [examples](https://github.com/tj/commander.js/tree/master/examples) directory.
## License
[MIT](https://github.com/tj/commander.js/blob/master/LICENSE)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,38 @@
{
"name": "commander",
"version": "2.20.3",
"description": "the complete solution for node.js command-line programs",
"keywords": [
"commander",
"command",
"option",
"parser"
],
"author": "TJ Holowaychuk <tj@vision-media.ca>",
"license": "MIT",
"repository": {
"type": "git",
"url": "https://github.com/tj/commander.js.git"
},
"scripts": {
"lint": "eslint index.js",
"test": "node test/run.js && npm run test-typings",
"test-typings": "tsc -p tsconfig.json"
},
"main": "index",
"files": [
"index.js",
"typings/index.d.ts"
],
"dependencies": {},
"devDependencies": {
"@types/node": "^12.7.8",
"eslint": "^6.4.0",
"should": "^13.2.3",
"sinon": "^7.5.0",
"standard": "^14.3.1",
"ts-node": "^8.4.1",
"typescript": "^3.6.3"
},
"typings": "typings/index.d.ts"
}

View File

@@ -0,0 +1,310 @@
// Type definitions for commander 2.11
// Project: https://github.com/visionmedia/commander.js
// Definitions by: Alan Agius <https://github.com/alan-agius4>, Marcelo Dezem <https://github.com/mdezem>, vvakame <https://github.com/vvakame>, Jules Randolph <https://github.com/sveinburne>
// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
declare namespace local {
class Option {
flags: string;
required: boolean;
optional: boolean;
bool: boolean;
short?: string;
long: string;
description: string;
/**
* Initialize a new `Option` with the given `flags` and `description`.
*
* @param {string} flags
* @param {string} [description]
*/
constructor(flags: string, description?: string);
}
class Command extends NodeJS.EventEmitter {
[key: string]: any;
args: string[];
/**
* Initialize a new `Command`.
*
* @param {string} [name]
*/
constructor(name?: string);
/**
* Set the program version to `str`.
*
* This method auto-registers the "-V, --version" flag
* which will print the version number when passed.
*
* @param {string} str
* @param {string} [flags]
* @returns {Command} for chaining
*/
version(str: string, flags?: string): Command;
/**
* Add command `name`.
*
* The `.action()` callback is invoked when the
* command `name` is specified via __ARGV__,
* and the remaining arguments are applied to the
* function for access.
*
* When the `name` is "*" an un-matched command
* will be passed as the first arg, followed by
* the rest of __ARGV__ remaining.
*
* @example
* program
* .version('0.0.1')
* .option('-C, --chdir <path>', 'change the working directory')
* .option('-c, --config <path>', 'set config path. defaults to ./deploy.conf')
* .option('-T, --no-tests', 'ignore test hook')
*
* program
* .command('setup')
* .description('run remote setup commands')
* .action(function() {
* console.log('setup');
* });
*
* program
* .command('exec <cmd>')
* .description('run the given remote command')
* .action(function(cmd) {
* console.log('exec "%s"', cmd);
* });
*
* program
* .command('teardown <dir> [otherDirs...]')
* .description('run teardown commands')
* .action(function(dir, otherDirs) {
* console.log('dir "%s"', dir);
* if (otherDirs) {
* otherDirs.forEach(function (oDir) {
* console.log('dir "%s"', oDir);
* });
* }
* });
*
* program
* .command('*')
* .description('deploy the given env')
* .action(function(env) {
* console.log('deploying "%s"', env);
* });
*
* program.parse(process.argv);
*
* @param {string} name
* @param {string} [desc] for git-style sub-commands
* @param {CommandOptions} [opts] command options
* @returns {Command} the new command
*/
command(name: string, desc?: string, opts?: commander.CommandOptions): Command;
/**
* Define argument syntax for the top-level command.
*
* @param {string} desc
* @returns {Command} for chaining
*/
arguments(desc: string): Command;
/**
* Parse expected `args`.
*
* For example `["[type]"]` becomes `[{ required: false, name: 'type' }]`.
*
* @param {string[]} args
* @returns {Command} for chaining
*/
parseExpectedArgs(args: string[]): Command;
/**
* Register callback `fn` for the command.
*
* @example
* program
* .command('help')
* .description('display verbose help')
* .action(function() {
* // output help here
* });
*
* @param {(...args: any[]) => void} fn
* @returns {Command} for chaining
*/
action(fn: (...args: any[]) => void): Command;
/**
* Define option with `flags`, `description` and optional
* coercion `fn`.
*
* The `flags` string should contain both the short and long flags,
* separated by comma, a pipe or space. The following are all valid
* all will output this way when `--help` is used.
*
* "-p, --pepper"
* "-p|--pepper"
* "-p --pepper"
*
* @example
* // simple boolean defaulting to false
* program.option('-p, --pepper', 'add pepper');
*
* --pepper
* program.pepper
* // => Boolean
*
* // simple boolean defaulting to true
* program.option('-C, --no-cheese', 'remove cheese');
*
* program.cheese
* // => true
*
* --no-cheese
* program.cheese
* // => false
*
* // required argument
* program.option('-C, --chdir <path>', 'change the working directory');
*
* --chdir /tmp
* program.chdir
* // => "/tmp"
*
* // optional argument
* program.option('-c, --cheese [type]', 'add cheese [marble]');
*
* @param {string} flags
* @param {string} [description]
* @param {((arg1: any, arg2: any) => void) | RegExp} [fn] function or default
* @param {*} [defaultValue]
* @returns {Command} for chaining
*/
option(flags: string, description?: string, fn?: ((arg1: any, arg2: any) => void) | RegExp, defaultValue?: any): Command;
option(flags: string, description?: string, defaultValue?: any): Command;
/**
* Allow unknown options on the command line.
*
* @param {boolean} [arg] if `true` or omitted, no error will be thrown for unknown options.
* @returns {Command} for chaining
*/
allowUnknownOption(arg?: boolean): Command;
/**
* Parse `argv`, settings options and invoking commands when defined.
*
* @param {string[]} argv
* @returns {Command} for chaining
*/
parse(argv: string[]): Command;
/**
* Parse options from `argv` returning `argv` void of these options.
*
* @param {string[]} argv
* @returns {ParseOptionsResult}
*/
parseOptions(argv: string[]): commander.ParseOptionsResult;
/**
* Return an object containing options as key-value pairs
*
* @returns {{[key: string]: any}}
*/
opts(): { [key: string]: any };
/**
* Set the description to `str`.
*
* @param {string} str
* @param {{[argName: string]: string}} argsDescription
* @return {(Command | string)}
*/
description(str: string, argsDescription?: {[argName: string]: string}): Command;
description(): string;
/**
* Set an alias for the command.
*
* @param {string} alias
* @return {(Command | string)}
*/
alias(alias: string): Command;
alias(): string;
/**
* Set or get the command usage.
*
* @param {string} str
* @return {(Command | string)}
*/
usage(str: string): Command;
usage(): string;
/**
* Set the name of the command.
*
* @param {string} str
* @return {Command}
*/
name(str: string): Command;
/**
* Get the name of the command.
*
* @return {string}
*/
name(): string;
/**
* Output help information for this command.
*
* @param {(str: string) => string} [cb]
*/
outputHelp(cb?: (str: string) => string): void;
/** Output help information and exit.
*
* @param {(str: string) => string} [cb]
*/
help(cb?: (str: string) => string): never;
}
}
declare namespace commander {
type Command = local.Command
type Option = local.Option
interface CommandOptions {
noHelp?: boolean;
isDefault?: boolean;
}
interface ParseOptionsResult {
args: string[];
unknown: string[];
}
interface CommanderStatic extends Command {
Command: typeof local.Command;
Option: typeof local.Option;
CommandOptions: CommandOptions;
ParseOptionsResult: ParseOptionsResult;
}
}
declare const commander: commander.CommanderStatic;
export = commander;

90
themes/keepit/node_modules/@algolia/cli/package.json generated vendored Normal file
View File

@@ -0,0 +1,90 @@
{
"name": "@algolia/cli",
"version": "4.0.8",
"description": "A Node CLI tools for manipulating data. Handy for day-to-day Algolia SE work.",
"license": "ISC",
"author": "Algolia, Inc. (https://www.algolia.com)",
"main": "index.js",
"repository": {
"type": "git",
"url": "git://github.com/algolia/algolia-cli.git"
},
"bin": {
"algolia": "./index.js"
},
"scripts": {
"test": "jest --runInBand",
"test:unit": "jest commands/",
"test:unit:watch": "jest --watch commands/",
"test:integration": "jest --runInBand tests/integration/",
"lint": "eslint .",
"lint:fix": "npm run lint -- --fix"
},
"engines": {
"node": ">=8.9.1",
"yarn": ">=1.10.1"
},
"files": [
"commands",
"commands.js",
"index.js",
"!commands/*.test.js"
],
"renovate": {
"extends": [
"config:js-app"
]
},
"keywords": [
"data",
"json",
"csv",
"manipulate",
"transform",
"process",
"parse",
"import",
"index",
"solutions",
"se",
"cli"
],
"dependencies": {
"JSONStream": "^1.3.5",
"algoliasearch": "^3.31.0",
"async": "^2.6.0",
"batch-stream": "^0.1.3",
"chalk": "^2.4.1",
"commander": "^2.19.0",
"csvtojson": "^2.0.8",
"inquirer": "^6.2.2",
"regex-parser": "^2.2.10",
"speedtest-net": "^1.5.1",
"stream-transform": "^1.0.7",
"through": "^2.3.8"
},
"devDependencies": {
"babel-eslint": "^10.0.1",
"babel-jest": "^23.6.0",
"dotenv": "^6.2.0",
"eslint": "^5.9.0",
"eslint-config-algolia": "^13.2.3",
"eslint-config-prettier": "^3.3.0",
"eslint-plugin-import": "^2.14.0",
"eslint-plugin-jest": "^22.1.2",
"eslint-plugin-prettier": "^3.0.0",
"jest": "^23.6.0",
"prettier": "^1.15.3",
"randomatic": "^3.1.1",
"rimraf": "^2.6.2",
"striptags": "^3.1.1"
},
"jest": {
"testEnvironment": "node",
"verbose": true,
"testURL": "http://localhost/",
"setupFiles": [
"./tests/config.js"
]
}
}

656
themes/keepit/node_modules/@algolia/cli/readme.md generated vendored Normal file
View File

@@ -0,0 +1,656 @@
# Algolia CLI
A Node CLI tool that makes it easy to perform common data manipulations and interactions with your Algolia app or indices.
- [Requirements](#requirements)
- [Install](#install)
- [Usage](#usage)
- [Commands](#commands)
- [Examples](#examples)
- [Contribute](#contribute)
# Requirements
- [Node.js](https://nodejs.org/)
# Install
- `npm install -g @algolia/cli`
# Usage
##### 📌 `algolia <COMMAND NAME> [OPTIONS]` 📌
```bash
$ algolia --help
$ algolia --version
$ algolia interactive
$ algolia search -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -q <query> -p <searchParams> -o <outputPath>
$ algolia import -s <sourceFilepath> -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -b <batchSize> -t <transformationFilepath> -m <maxconcurrency> -p <csvToJsonParams>
$ algolia export -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -o <outputPath> -p <algoliaParams>
$ algolia getsettings -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName>
$ algolia setsettings -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -s <sourceFilepath> -p <setSettingsParams>
$ algolia addrules -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -s <sourceFilepath> -p <batchRulesParams>
$ algolia exportrules -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -o <outputPath>
$ algolia addsynonyms -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -s <sourceFilepath> -p <batchSynonymsParams>
$ algolia exportsynonyms -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -o <outputPath>
$ algolia transferindex -a <sourcealgoliaAppId> -k <sourcealgoliaApiKey> -n <sourcealgoliaIndexName> -d <destinationAlgoliaAppId> -y <destinationAlgoliaApiKey> -i <destinationIndexName> -t <transformationFilepath> -e <true|false>
$ algolia transferindexconfig -a <sourcealgoliaAppId> -k <sourcealgoliaApiKey> -n <sourcealgoliaIndexName> -d <destinationAlgoliaAppId> -y <destinationAlgoliaApiKey> -i <destinationIndexName> -p <configParams> -e <true|false>
$ algolia deleteindicespattern -a <algoliaAppId> -k <algoliaApiKey> -r '<regexp>' -x <true|false>
$ algolia transformlines -s <sourceFilepath> -o <outputPath> -t <transformationFilepath>
$ algolia examples
```
See also [additional examples](#examples).
# Commands
### 1. Help | `--help`
##### Description:
Get basic usage info for all provided CLI scripts.
##### Usage:
```shell
algolia --help
```
or
```
algolia -h
```
### 2. Version | `--version`
##### Description:
Get version info for npm package.
##### Usage:
```shell
algolia --version
```
or
```
algolia -v
```
### 3. Interactive | `interactive`
##### Description:
Use Algolia CLI in interactive mode. Get command and argument prompts.
##### Usage:
```shell
algolia interactive
```
### 4. Search | `search`
##### Description:
Search an Algolia index.
##### Usage:
```shell
algolia search -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -q <query> -p <searchParams> -o <outputPath>
```
##### Options:
- `<algoliaAppId>` | Required
- `<algoliaApiKey>` | Required
- `<algoliaIndexName>` | Required
- `<query>` | Optional | Search query string to send to Algolia index. Defaults to `''`.
- `<searchParams>` | Optional | JSON params to be passed to Algolia `.search()` [method](https://www.algolia.com/doc/api-reference/api-methods/search/?language=javascript).
- `<outputPath>` | Optional | Local path where search results file will be saved.
##### Notes:
- If no `<outputPath>` is provided, command will simply console.log() the response.
- If an `<outputPath>` is provided, command will write a JSON file to that location.
- Provided `<outputPath>` path must include file name.
- See [search parameters](https://www.algolia.com/doc/api-reference/search-api-parameters/) for more documentation about search options.
### 5. Import | `import`
##### Description:
Import JSON or CSV data into Algolia index, from a file or directory of files.
You may also optionally apply custom transformations to each object indexed. CSV files will automatically be converted to JSON before transformations are applied.
Will handle arbitrarily large files without performance issues.
##### Usage:
```shell
algolia import -s <sourceFilepath> -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -b <batchSize> -t <transformationFilepath> -m <maxConcurrency> -p <csvToJsonParams>
```
##### Options:
- `<sourceFilepath>` | Required | Path to a JSON or CSV file, or to a directory of such files.
- `<algoliaAppId>` | Required
- `<algoliaApiKey>` | Required
- `<algoliaIndexName>` | Required
- `<batchSize>` | Optional | Number of JSON objects to be included in each batch for indexing. Default is `5000`.
- `<transformationFilepath>` | Optional | The path to any file that exports a function which (1) takes 2 arguments; an object and a callback, then (2) ends by calling said callback with the 2 arguments `null` and `<YOUR_TRANSFORMED_OBJECT>`.
- `<maxConcurrency>` | Optional | Maximum number of concurrent filestreams to process. Default is `2`.
- `<csvToJsonParams>` | Optional | Stringified [Parser parameters](https://github.com/Keyang/node-csvtojson#parameters) object passed to [csvtojson](https://www.npmjs.com/package/csvtojson) module.
##### Example Transformation File:
See `transformations/example-transformations.js` for an extensive JSON object transformation example.
Simple transformation file example:
```javascript
module.exports = (data,cb) => {
try {
const record = Object.assign({}, data);
record.objectID = data.product_id;
record.score = Math.floor(Math.random() * 100);
record.formattedNumber = parseInt(data.integer_formatted_as_string, 10);
cb(null, record);
} catch (e) {
console.log('Transformation error:', e.message, e.stack);
throw e;
}
}
```
##### Notes:
- `<sourceFilepath>` may target a file or a directory of files.
- JSON files must contain an array of objects.
- CSV files must have a `.csv` extension.
- `<transformationFilepath>` requires a path to a transformation file. See [example file](transformations/example-transformations.js).
- Make sure you only import JSON or CSV files. Don't accidentally try to import hidden files like `.DS_Store`, log files, etc. as they will throw an error.
- Command assumes each file contains an array of JSON objects unless the file extension ends with `.csv`.
- CSV to JSON conversion performed using [csvtojson](https://www.npmjs.com/package/csvtojson) package.
- If no `<batchSize>` is explicitly provided, command will try to determine optimal batch size by estimating average record size, estimating network speed, and calculating a size that should work well given the concurrency.
- If command outputs a `AlgoliaSearchRequestTimeoutError` error, this means a batch of records failed to import. This typically occurs when attempting to import too much data over too slow a network connection. Command will automatically attempt to reduce `<batchSize>` to compensate, and re-try. If issues persist, consider reducing `<maxConcurrency>` and/or `<batchSize>`.
- If command outputs a `High memory usage` warning, it means the process is consuming a very high percentage of the estimated system heap allocation for the node process. Command will automatically attempt to reduce `<batchSize>` to compensate. If issues persist, consider reducing `<maxConcurrency>` and/or `<batchSize>`.
### 6. Export | `export`
##### Description:
Download all JSON records from a specific Algolia index.
##### Usage:
```shell
algolia export -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -o <outputPath> -p <algoliaParams>
```
##### Options:
- `<algoliaAppId>` | Required
- `<algoliaApiKey>` | Required
- `<algoliaIndexName>` | Required
- `<outputPath>` | Optional | Path to an existing local directory where output files will be saved (filenames are autogenerated). If no output path is provided, defaults to current working directory.
- `<algoliaParams>` | Optional | JSON [Search params](https://www.algolia.com/doc/api-reference/search-api-parameters/) object passed to `browseAll()` [method](https://www.algolia.com/doc/api-reference/api-methods/browse/).
##### Notes:
- `<outputPath>` must be a directory.
### 7. Get Settings | `getsettings`
##### Description:
Get settings for a specific Algolia index.
##### Usage:
```shell
algolia getsettings -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName>
```
##### Options:
- `<algoliaAppId>` | Required
- `<algoliaApiKey>` | Required
- `<algoliaIndexName>` | Required
##### Notes:
- To write settings JSON locally, just redirect the output to a file. For example:
`$ algolia getsettings -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME > ~/Desktop/EXAMPLE_FILE_NAME.json`
### 8. Set Settings | `setsettings`
##### Description:
Set settings for a specific Algolia index.
##### Usage:
```shell
algolia setsettings -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -s <sourceFilepath> -p <setSettingsParams>
```
##### Options:
- `<algoliaAppId>` | Required
- `<algoliaApiKey>` | Required
- `<algoliaIndexName>` | Required
- `<sourceFilepath>` | Required | Path to a JSON file containing a settings object.
- `<setSettingsParams>` | Optional | JSON object containing options passed to `setSettings()` [method](https://www.algolia.com/doc/api-reference/api-methods/set-settings/).
##### Example settings file:
```js
module.exports = {
minWordSizefor1Typo: 4,
minWordSizefor2Typos: 8,
hitsPerPage: 20,
maxValuesPerFacet: 100,
version: 2,
attributesToIndex: null,
numericAttributesToIndex: null,
attributesToRetrieve: null,
unretrievableAttributes: null,
optionalWords: null,
attributesForFaceting: null,
attributesToSnippet: null,
attributesToHighlight: null,
paginationLimitedTo: 1000,
attributeForDistinct: null,
exactOnSingleWordQuery: 'attribute',
ranking:
[ 'typo',
'geo',
'words',
'filters',
'proximity',
'attribute',
'exact',
'custom' ],
customRanking: null,
separatorsToIndex: '',
removeWordsIfNoResults: 'none',
queryType: 'prefixLast',
highlightPreTag: '<em>',
highlightPostTag: '</em>',
snippetEllipsisText: '',
alternativesAsExact: [ 'ignorePlurals', 'singleWordSynonym' ]
};
```
##### Example setSettings params:
```
'{"forwardToReplicas":true}'
```
##### Notes:
- Any index setting parameter needs to be added directly in the file containing the settings object. See [Settings API paraameters documentation](https://www.algolia.com/doc/api-reference/settings-api-parameters/) to find the full list of index settings parameters.
- forwardToReplicas is currently the only option that can be passed to the settings method as an optional <setSettingsParams> argument.
### 9. Add Rules | `addrules`
##### Description:
Import a local JSON file of query rules to an Algolia index.
##### Usage:
```shell
algolia addrules -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -s <sourceFilepath> -p <batchRulesParams>
```
##### Options:
- `<algoliaAppId>` | Required
- `<algoliaApiKey>` | Required
- `<algoliaIndexName>` | Required
- `<sourceFilepath>` | Required | Path to a JSON file containing an array of query rule objects.
- `<batchRulesParams>` | Optional | JSON object containing options passed to `batchRules()` [method](https://www.algolia.com/doc/api-reference/api-methods/batch-rules/).
##### Notes:
- See [batchRules documentation](https://www.algolia.com/doc/api-reference/api-methods/batch-rules/) and [implementing query rules documentation](https://www.algolia.com/doc/guides/managing-results/refine-results/merchandising-and-promoting/in-depth/implementing-query-rules/) for more info.
### 10. Export Rules | `exportrules`
##### Description:
Download all query rules from a specific Algolia index.
##### Usage:
```shell
algolia exportrules -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -o <outputPath>
```
##### Options:
- `<algoliaAppId>` | Required
- `<algoliaApiKey>` | Required
- `<algoliaIndexName>` | Required
- `<outputPath>` | Optional | Local path where query rules file will be saved. If no output path is provided, defaults to current working directory.
##### Notes:
- `<outputPath>`path must include file name.
### 11. Add Synonyms | `addsynonyms`
##### Description:
Import a local CSV or JSON file of synonyms to an Algolia index. Some public synonym files can be downloaded from [this repository](https://github.com/algolia/synonym-dictionaries). Disclaimer: These are not intended to be all encompassing -- edits may be needed for your use case.
Note that if importing a CSV file, the expected format is file with no headers and with each row of comma-separated values being a group of synonyms for each other. For more information, read our [documentation](https://www.algolia.com/doc/guides/managing-results/optimize-search-results/adding-synonyms/how-to/managing-synonyms-from-the-dashboard/#csv) on the topic.
##### Usage:
```shell
algolia addsynonyms -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -s <sourceFilepath> -p <batchSynonymsParams>
```
##### Options:
- `<algoliaAppId>` | Required
- `<algoliaApiKey>` | Required
- `<algoliaIndexName>` | Required
- `<sourceFilepath>` | Required | Path to a CSV or JSON file containing an array of synonyms objects.
- `<batchSynonymsParams>` | Optional | JSON object containing options passed to `batchSynonyms()` [method](https://www.algolia.com/doc/api-reference/api-methods/batch-synonyms/).
##### Notes:
- See [batchSynonyms documentation](https://www.algolia.com/doc/api-reference/api-methods/batch-synonyms/) and [adding synonyms documentation](https://www.algolia.com/doc/guides/managing-results/optimize-search-results/adding-synonyms/) for more info.
### 12. Export Synonyms | `exportsynonyms`
##### Description:
Download all synonyms from a specific Algolia index.
##### Usage:
```shell
algolia exportsynonyms -a <algoliaAppId> -k <algoliaApiKey> -n <algoliaIndexName> -o <outputPath>
```
##### Options:
- `<algoliaAppId>` | Required
- `<algoliaApiKey>` | Required
- `<algoliaIndexName>` | Required
- `<outputPath>` | Optional | Local path where synonyms file will be saved. If no output path is provided, defaults to current working directory.
##### Notes:
- `<outputPath>`path must include file name.
### 13. Transfer Index | `transferindex`
##### Description:
Transfer all data and settings (including synonyms and query rules) from one Algolia app/index to another.
##### Usage:
```shell
algolia transferindex -a <sourceAlgoliaAppId> -k <sourceAlgoliaApiKey> -n <sourceAlgoliaIndexName> -d <destinationAlgoliaAppId> -y <destinationAlgoliaApiKey> -i <destinationIndexName> -t <transformationFilepath> -e <excludeReplicas>
```
##### Options:
- `<sourceAlgoliaAppId>` | Required
- `<sourceAlgoliaApiKey>` | Required
- `<sourceAlgoliaIndexName>` | Required
- `<destinationAlgoliaAppId>` | Required
- `<destinationAlgoliaApiKey>` | Required
- `<destinationIndexName>` | Optional | If no destination index name is specified, script will default to creating a new index with the same name as the source index.
- `<transformationFilepath>` | Optional | The path to any file that exports a function which (1) takes a single object as argument, then (2) returns a transformed object.
- `<excludeReplicas>` | Optional | This is a boolean. When `true`, it will exclude the `replicas` setting when copying settings to the destination index. When `false`, it will copy the full settings object. Defaults to `false`.
##### Example Transformation File:
Simple transformation file for transferring an index:
```javascript
module.exports = (obj) => {
try {
const record = {};
record.objectID = obj.product_id;
record.score = Math.floor(Math.random() * 100);
record.formattedNumber = parseInt(obj.integer_formatted_as_string, 10);
} catch (e) {
console.log('Transformation error:', e.message, e.stack);
throw e;
}
}
```
##### Notes:
- Command duplicates data and copies settings, synonyms, and rules; does not delete or affect source index.
- Command does NOT forward settings or synonyms to replicas.
### 14. Transfer Index Config | `transferindexconfig`
##### Description:
Transfer an index's settings, synonyms, and query rules to another index. Works even across indices in different Algolia applications.
##### Usage:
```shell
algolia transferindexconfig -a <sourceAlgoliaAppId> -k <sourceAlgoliaApiKey> -n <sourceAlgoliaIndexName> -d <destinationAlgoliaAppId> -y <destinationAlgoliaApiKey> -i <destinationIndexName> -p <configParams> -e <excludeReplicas>
```
##### Options:
- `<sourceAlgoliaAppId>` | Required
- `<sourceAlgoliaApiKey>` | Required
- `<sourceAlgoliaIndexName>` | Required
- `<destinationAlgoliaAppId>` | Required
- `<destinationAlgoliaApiKey>` | Required
- `<destinationIndexName>` | Optional | If no destination index name is specified, script will default to targetting an existing index with the same name as the source index.
- `<configParams>` | Optional | JSON object containing one or both of the following two properties: `batchSynonymsParams` and `batchRulesParams`. Each of those property values may contain a parameters object to be passed to the [batchSynonyms](https://www.algolia.com/doc/api-reference/api-methods/batch-synonyms/) and [batchRules](https://www.algolia.com/doc/api-reference/api-methods/batch-rules/) respectively.
- `<excludeReplicas>` | Optional | This is a boolean. When `true`, it will exclude the `replicas` setting when copying settings to the destination index. When `false`, it will copy the full settings object. Defaults to `false`.
##### Notes:
- When transferring synonyms and query rules, `forwardToReplicas`, `replaceExistingSynonyms`, and `clearExistingRules` params will default to false, unless you specify `<configParams>`.
### 15. Delete Indices Pattern | `deleteindicespattern`
##### Description:
Delete multiple indices at once (main or replica indices included) using a regular expression.
##### Usage:
```shell
algolia deleteindicespattern -a <algoliaAppId> -k <algoliaApiKey> -r '<regexp>' -x <dryrun>
```
##### Options:
- `<algoliaAppId>` | Required
- `<algoliaApiKey>` | Required
- `<regexp>` | Required | Provide regexes without the leading and trailing slashes
- `<dryrun>` | Required | This is a boolean. When `true` it will run in dry mode and show what will be deleted, when `false` it will really delete the indices. Careful!
##### Notes:
- The command handles replicas. First it update the settings of all main indices removing any replica that will match the regular expression. Then it will delete all matching indices (main and replica indices).
##### Example:
```shell
algolia deleteindicespattern -a someAppId -k someApiKey -r '^staging__' -x false
```
This will delete all indices of the application that are starting with "staging__".
### 16. Transform Lines | `transformlines`
##### Description:
Transform a file line-by-line.
##### Usage:
```shell
algolia transformlines -s <sourceFilepath> -o <outputPath> -t <transformationFilepath>
```
##### Options:
- `<sourceFilepath>` | Required | Path to a single `.js` or `.json` file OR a directory of such files.
- `<outputPath>` | Optional | Path to an existing local directory where output files will be saved (saved output filenames will match corresponding source filenames). If no output path is provided, defaults to current working directory.
- `<transformationFilepath>` | Optional | Path to file that exports a function which (1) takes a line string, and (2) returns a transformed line string.
##### Example use case:
Mapping each line of input file to a new output file.
Originally designed for converting `.json-seq` files to regular comma separated JSON arrays, in order to index them with the `import` cli tool.
##### Example Transformation File:
Let's say we had this source JSON file:
```json
[
{"id":1,"color":"blue"},
{"id":2,"color":"red"},
{"id":3,"color":"green"}
]
```
and we wanted to filter out any objects that didn't have a "color" value of "blue". In this case, our transformations function could be something like this:
```javascript
module.exports = (line) => {
if (line === '[' || line === ']') {
return line;
} else if (line.includes('"color":"blue"')) {
return line;
} else {
return '\n';
}
}
```
##### Notes:
- `<outputPath>` must be a directory.
- Running `transformlines` command without providing optional `<transformationFilepath>` param will cause it to assume it's parsing a `.json-seq` file; thus, it will apply the `defaultLineTransformation` method in `transformLines.js` to each line. This checks each line for the ASCII Record Separator character `\u001e` and replaces it with a `,`. It will _also_ cause it to enclose the whole file in "[" and "]" square brackets to make it a valid JS array. Providing a custom transformation method via the optional `<transformationFilepath>` param will make it exclusively run your transformation function instead of the default one (and in this case it will also omit adding enclosing square brackets).
### 14. Examples | `examples`
##### Description:
Display command usage examples.
##### Usage:
```shell
algolia examples
```
##### Notes:
- See equivalent list of [examples below](#examples).
# Examples
```bash
$ algolia --help
$ algolia --version
$ algolia interactive
$ algolia search -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -q 'example query' -p '{"facetFilters":["category:book"]}' -o ~/Desktop/results.json
$ algolia import -s ~/Desktop/example_source_directory/ -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -b 5000 -t ~/Desktop/example_transformations.js -m 4 -p '{"delimiter":[":"]}'
$ algolia export -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -o ~/Desktop/example_output_folder/ -p '{"filters":["category:book"]}'
$ algolia getsettings -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME
$ algolia setsettings -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -s ~/Desktop/example_settings.json -p '{"forwardToReplicas":true}'
$ algolia addrules -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -s ~/Desktop/example_rules.json -p '{"forwardToReplicas":false,"clearExistingRules":true}'
$ algolia exportrules -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -o ~/Desktop/example_rules.json
$ algolia addsynonyms -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -s ~/Desktop/example_synonyms.json -p '{"forwardToReplicas":true,"clearExistingSynonyms":true}'
$ algolia exportsynonyms -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -n EXAMPLE_INDEX_NAME -o ~/Desktop/example_synonyms.json
$ algolia transferindex -a EXAMPLE_SOURCE_APP_ID -k EXAMPLE_SOURCE_API_KEY -n EXAMPLE_SOURCE_INDEX_NAME -d EXAMPLE_DESTINATION_APP_ID -y EXAMPLE_DESTINATION_API_KEY -i EXAMPLE_DESTINATION_INDEX_NAME -t ~/Desktop/example_transformations.js -e true
$ algolia transferindexconfig -a EXAMPLE_SOURCE_APP_ID -k EXAMPLE_SOURCE_API_KEY -n EXAMPLE_SOURCE_INDEX_NAME -d EXAMPLE_DESTINATION_APP_ID -y EXAMPLE_DESTINATION_API_KEY -i EXAMPLE_DESTINATION_INDEX_NAME -p '{"batchSynonymsParams":{"forwardToReplicas":true,"replaceExistingSynonyms":true},"batchRulesParams":{"forwardToReplicas":true,"clearExistingRules":true}}' -e true
$ algolia deleteindicespattern -a EXAMPLE_APP_ID -k EXAMPLE_API_KEY -r '^regex' -x true
$ algolia transformlines -s ~/Desktop/example_source_file.json -o ~/Desktop/example_output_folder/ -t ~/Desktop/example_transformations.js
$ algolia examples
```
# Contribute
## Requirements
- Node: `brew install node` or [Node docs](https://nodejs.org/en/)
- Yarn: `brew install yarn` or [Yarn docs](https://yarnpkg.com/lang/en/)
## Install
- Clone repo.
- `yarn install`
- Create `.env` file in project root and assign environment variables as listed [below](#environment-variables).
## Environment variables
- `ALGOLIA_TEST_APP_ID`
- `ALGOLIA_TEST_API_KEY`
- `ALGOLIA_TEST_INDEX_NAME`
- `ALGOLIA_TEST_ALT_APP_ID`
- `ALGOLIA_TEST_ALT_API_KEY`
## Develop
- Run `node index.js <command_name> [options]` to test various commands/options.
- Write code!
- Please use [git-flow](https://github.com/nvie/gitflow) and commit your changes on a feature branch, rebase it on develop branch before finishing the feature, then issue pull request to develop branch
## Tests
- `yarn test` to run full test suite locally
- `yarn test:unit` to run unit test suite only
- `yarn test:unit:watch` to run unit test suite with interactive `--watch` flag
- `yarn test:integration` to run integration test suite only
## Lint
- `yarn lint` to run eslint
- `yarn lint:fix` to run eslint with --fix flag