Multipart upload directory #243

Merged
bones7242 merged 5 commits from multipart-upload-directory into upload-refactor 2017-11-07 01:05:04 +01:00
19 changed files with 72 additions and 116 deletions

1
.gitignore vendored
View file

@ -1,3 +1,4 @@
node_modules node_modules
.idea .idea
config/config.json config/config.json
config/speechConfig.js

View file

@ -14,13 +14,10 @@ spee.ch is a single-serving site that reads and publishes images and videos to a
* start spee.ch * start spee.ch
* clone this repo * clone this repo
* run `npm install` * run `npm install`
* to start the server, from your command line run `node speech.js` while passing three environmental variables: * create your `speechConfig.js` file
* (1) your lbry wallet address (`LBRY_CLAIM_ADDRESS`), * copy `speechConfig_example.js` and name it `speechConfig.js`
* (2) your mysql username (`MYSQL_USERNAME`), * replace the `null` values in the config file with the appropriate values for your environement
* (2) your mysql password (`MYSQL_PASSWORD`), * to start the server, from your command line run `node speech.js`
* (3) the environment to run (`NODE_ENV`).
* i.e. `LBRY_CLAIM_ADDRESS=<your wallet address here> MYSQL_USERNAME=<username here> MYSQL_PASSWORD=<password here> NODE_ENV=development node speech.js`
* e.g. `LBRY_CLAIM_ADDRESS=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX MYSQL_USERNAME="lbry" MYSQL_PASSWORD="xxxxxx" NODE_ENV=development node speech.js`
* To run hot, use `nodemon` instead of `node` * To run hot, use `nodemon` instead of `node`
* visit [localhost:3000](http://localhost:3000) * visit [localhost:3000](http://localhost:3000)

View file

@ -1,15 +0,0 @@
{
"WalletConfig": {
"LbryClaimAddress": "LBRY_CLAIM_ADDRESS"
},
"Database": {
"Username": "MYSQL_USERNAME",
"Password": "MYSQL_PASSWORD"
},
"Logging": {
"SlackWebHook": "SLACK_WEB_HOOK"
},
"Session": {
"SessionKey": "SESSION_KEY"
}
}

View file

@ -1 +0,0 @@
{}

View file

@ -1,23 +0,0 @@
{
"WalletConfig": {
"LbryClaimAddress": null,
"DefaultChannel": null
},
"AnalyticsConfig":{
"GoogleId": null
},
"Database": {
"Database": "lbry",
"Username": null,
"Password": null
},
"Logging": {
"LogLevel": null,
"SlackWebHook": null,
"SlackErrorChannel": null,
"SlackInfoChannel": null
},
"Session": {
"SessionKey": null
}
}

View file

@ -1,13 +0,0 @@
{
"WalletConfig": {
"DefaultChannel": "@speechDev"
},
"AnalyticsConfig":{
"GoogleId": "UA-100747990-1"
},
"Logging": {
"LogLevel": "silly",
"SlackErrorChannel": "#staging_speech-errors",
"SlackInfoChannel": "none"
}
}

View file

@ -1,13 +0,0 @@
{
"WalletConfig": {
"DefaultChannel": "@speech"
},
"AnalyticsConfig":{
"GoogleId": "UA-60403362-3"
},
"Logging": {
"LogLevel": "verbose",
"SlackErrorChannel": "#speech-errors",
"SlackInfoChannel": "#speech-logs"
}
}

View file

@ -1,29 +1,26 @@
const config = require('config'); const config = require('./speechConfig.js');
const SLACK_WEB_HOOK = config.get('Logging.SlackWebHook');
const SLACK_ERROR_CHANNEL = config.get('Logging.SlackErrorChannel');
const SLACK_INFO_CHANNEL = config.get('Logging.SlackInfoChannel');
const winstonSlackWebHook = require('winston-slack-webhook').SlackWebHook; const winstonSlackWebHook = require('winston-slack-webhook').SlackWebHook;
module.exports = (winston) => { module.exports = (winston) => {
if (SLACK_WEB_HOOK) { if (config.logging.slackWebHook) {
// add a transport for errors to slack // add a transport for errors to slack
winston.add(winstonSlackWebHook, { winston.add(winstonSlackWebHook, {
name : 'slack-errors-transport', name : 'slack-errors-transport',
level : 'error', level : 'error',
webhookUrl: SLACK_WEB_HOOK, webhookUrl: config.logging.slackWebHook,
channel : SLACK_ERROR_CHANNEL, channel : config.logging.slackErrorChannel,
username : 'spee.ch', username : 'spee.ch',
iconEmoji : ':face_with_head_bandage:', iconEmoji : ':face_with_head_bandage:',
}); });
winston.add(winstonSlackWebHook, { winston.add(winstonSlackWebHook, {
name : 'slack-info-transport', name : 'slack-info-transport',
level : 'info', level : 'info',
webhookUrl: SLACK_WEB_HOOK, webhookUrl: config.logging.slackWebHook,
channel : SLACK_INFO_CHANNEL, channel : config.logging.slackInfoChannel,
username : 'spee.ch', username : 'spee.ch',
iconEmoji : ':nerd_face:', iconEmoji : ':nerd_face:',
}); });
// send test message // send test message
winston.error('Slack error logging is online.'); winston.error('Slack error logging is online.');
winston.info('Slack info logging is online.'); winston.info('Slack info logging is online.');
} else { } else {

View file

@ -0,0 +1,22 @@
module.exports = {
wallet: {
lbryClaimAddress: null, // choose an address from your lbry wallet
},
analytics: {
googleId: null, // google id for analytics tracking; leave `null` if not applicable
},
sql: {
database: null, // name of mysql database
username: null, // username for mysql
password: null, // password for mysql
},
logging: {
logLevel : null, // options: silly, debug, verbose, info
slackWebHook : null, // enter a webhook if you wish to push logs to slack; otherwise leave as `null`
slackErrorChannel: null, // enter a slack channel (#example) for errors to be sent to; otherwise leave null
slackInfoChannel : null, // enter a slack channel (#info) for info level logs to be sent to otherwise leave null
},
session: {
sessionKey: null, // enter a secret key to be used for session encryption
},
};

View file

@ -6,13 +6,13 @@ const publishHelpers = require('../helpers/publishHelpers.js');
module.exports = { module.exports = {
publish (publishParams, fileName, fileType) { publish (publishParams, fileName, fileType) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let publishResults = {}; let publishResults, certificateId, channelName;
// 1. publish the file // publish the file
return lbryApi.publishClaim(publishParams) return lbryApi.publishClaim(publishParams)
// 2. upsert File record (update is in case the claim has been published before by this daemon)
.then(tx => { .then(tx => {
logger.info(`Successfully published ${fileName}`, tx); logger.info(`Successfully published ${fileName}`, tx);
publishResults = tx; publishResults = tx;
// get the channel information
if (publishParams.channel_name) { if (publishParams.channel_name) {
logger.debug(`this claim was published in channel: ${publishParams.channel_name}`); logger.debug(`this claim was published in channel: ${publishParams.channel_name}`);
return db.Channel.findOne({where: {channelName: publishParams.channel_name}}); return db.Channel.findOne({where: {channelName: publishParams.channel_name}});
@ -22,13 +22,17 @@ module.exports = {
} }
}) })
.then(channel => { .then(channel => {
let certificateId = null; // set channel information
let channelName = null; certificateId = null;
channelName = null;
if (channel) { if (channel) {
certificateId = channel.channelClaimId; certificateId = channel.channelClaimId;
channelName = channel.channelName; channelName = channel.channelName;
} }
logger.debug(`certificateId: ${certificateId}`); logger.debug(`certificateId: ${certificateId}`);
})
.then(() => {
// create the File record
const fileRecord = { const fileRecord = {
name : publishParams.name, name : publishParams.name,
claimId : publishResults.claim_id, claimId : publishResults.claim_id,
@ -42,6 +46,7 @@ module.exports = {
fileType, fileType,
nsfw : publishParams.metadata.nsfw, nsfw : publishParams.metadata.nsfw,
}; };
// create the Claim record
const claimRecord = { const claimRecord = {
name : publishParams.name, name : publishParams.name,
claimId : publishResults.claim_id, claimId : publishResults.claim_id,
@ -57,11 +62,12 @@ module.exports = {
certificateId, certificateId,
channelName, channelName,
}; };
// upsert criteria
const upsertCriteria = { const upsertCriteria = {
name : publishParams.name, name : publishParams.name,
claimId: publishResults.claim_id, claimId: publishResults.claim_id,
}; };
// create the records // upsert the records
return Promise.all([db.upsert(db.File, fileRecord, upsertCriteria, 'File'), db.upsert(db.Claim, claimRecord, upsertCriteria, 'Claim')]); return Promise.all([db.upsert(db.File, fileRecord, upsertCriteria, 'File'), db.upsert(db.Claim, claimRecord, upsertCriteria, 'Claim')]);
}) })
.then(([file, claim]) => { .then(([file, claim]) => {

View file

@ -1,8 +1,8 @@
const logger = require('winston'); const logger = require('winston');
const ua = require('universal-analytics'); const ua = require('universal-analytics');
const config = require('config'); const config = require('../config/speechConfig.js');
const db = require('../models'); const db = require('../models');
const googleApiKey = config.get('AnalyticsConfig.GoogleId'); const googleApiKey = config.analytics.googleId;
module.exports = { module.exports = {
postToStats (action, url, ipAddress, name, claimId, result) { postToStats (action, url, ipAddress, name, claimId, result) {

View file

@ -1,15 +1,13 @@
const config = require('config'); const config = require('../config/speechConfig.js');
const logger = require('winston'); const logger = require('winston');
const fs = require('fs');
module.exports = function () { module.exports = function () {
// get the config file // get the config file
const defaultConfigFile = JSON.parse(fs.readFileSync('./config/default.json'));
for (let configCategoryKey in defaultConfigFile) { for (let configCategoryKey in config) {
if (defaultConfigFile.hasOwnProperty(configCategoryKey)) { if (config.hasOwnProperty(configCategoryKey)) {
// get the final variables for each config category // get the final variables for each config category
const configVariables = config.get(configCategoryKey); const configVariables = config[configCategoryKey];
for (let configVarKey in configVariables) { for (let configVarKey in configVariables) {
if (configVariables.hasOwnProperty(configVarKey)) { if (configVariables.hasOwnProperty(configVarKey)) {
// print each variable // print each variable

View file

@ -1,10 +1,10 @@
const Handlebars = require('handlebars'); const Handlebars = require('handlebars');
const config = require('config'); const config = require('../config/speechConfig.js');
module.exports = { module.exports = {
// define any extra helpers you may need // define any extra helpers you may need
googleAnalytics () { googleAnalytics () {
const googleApiKey = config.get('AnalyticsConfig.GoogleId'); const googleApiKey = config.analytics.googleId;
return new Handlebars.SafeString( return new Handlebars.SafeString(
`<script> `<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){

View file

@ -1,7 +1,7 @@
const logger = require('winston'); const logger = require('winston');
const fs = require('fs'); const fs = require('fs');
const db = require('../models'); const db = require('../models');
const config = require('config'); const config = require('../config/speechConfig.js');
module.exports = { module.exports = {
validateApiPublishRequest (body, files) { validateApiPublishRequest (body, files) {
@ -110,7 +110,7 @@ module.exports = {
license, license,
nsfw, nsfw,
}, },
claim_address: config.get('WalletConfig.LbryClaimAddress'), claim_address: config.wallet.lbryClaimAddress,
}; };
// add thumbnail to channel if video // add thumbnail to channel if video
if (thumbnail !== null) { if (thumbnail !== null) {
@ -137,7 +137,7 @@ module.exports = {
db.File.findAll({ where: { name } }) db.File.findAll({ where: { name } })
.then(result => { .then(result => {
if (result.length >= 1) { if (result.length >= 1) {
const claimAddress = config.get('WalletConfig.LbryClaimAddress'); const claimAddress = config.wallet.lbryClaimAddress;
// filter out any results that were not published from spee.ch's wallet address // filter out any results that were not published from spee.ch's wallet address
const filteredResult = result.filter((claim) => { const filteredResult = result.filter((claim) => {
return (claim.address === claimAddress); return (claim.address === claimAddress);

View file

@ -2,13 +2,13 @@ const fs = require('fs');
const path = require('path'); const path = require('path');
const Sequelize = require('sequelize'); const Sequelize = require('sequelize');
const basename = path.basename(module.filename); const basename = path.basename(module.filename);
const config = require('config'); const config = require('../config/speechConfig.js');
const db = {}; const db = {};
const logger = require('winston'); const logger = require('winston');
const database = config.get('Database.Database'); const database = config.sql.database;
const username = config.get('Database.Username'); const username = config.sql.username;
const password = config.get('Database.Password'); const password = config.sql.password;
const sequelize = new Sequelize(database, username, password, { const sequelize = new Sequelize(database, username, password, {
host : 'localhost', host : 'localhost',
@ -66,7 +66,7 @@ db.upsert = (Model, values, condition, tableName) => {
} }
}) })
.catch(function (error) { .catch(function (error) {
logger.error('Sequelize findOne error', error); logger.error(`${tableName}.upsert error`, error);
}); });
}; };

View file

@ -34,7 +34,6 @@
"cookie-session": "^2.0.0-beta.3", "cookie-session": "^2.0.0-beta.3",
"express": "^4.15.2", "express": "^4.15.2",
"express-handlebars": "^3.0.0", "express-handlebars": "^3.0.0",
"express-session": "^1.15.5",
"form-data": "^2.3.1", "form-data": "^2.3.1",
"helmet": "^3.8.1", "helmet": "^3.8.1",
"mysql2": "^1.3.5", "mysql2": "^1.3.5",

View file

@ -175,7 +175,7 @@ var publishFileFunctions = {
}, },
showUploadProgressMessage: function (percentage){ showUploadProgressMessage: function (percentage){
this.updatePublishStatus('<p>File is loading to server</p>'); this.updatePublishStatus('<p>File is loading to server</p>');
this.updateUploadPercent('<p class="blue">' + percentage + '</p>'); this.updateUploadPercent('<p class="blue">' + percentage + '% </p>');
}, },
showFilePublishUpdate: function (msg) { showFilePublishUpdate: function (msg) {
this.updatePublishStatus('<p>' + msg + '</p>'); this.updatePublishStatus('<p>' + msg + '</p>');

View file

@ -1,6 +1,6 @@
const logger = require('winston'); const logger = require('winston');
const multipart = require('connect-multiparty'); const multipart = require('connect-multiparty');
const multipartMiddleware = multipart(); const multipartMiddleware = multipart({uploadDir: '/home/lbry/test/'});
const db = require('../models'); const db = require('../models');
const { publish } = require('../controllers/publishController.js'); const { publish } = require('../controllers/publishController.js');
const { getClaimList, resolveUri } = require('../helpers/lbryApi.js'); const { getClaimList, resolveUri } = require('../helpers/lbryApi.js');
@ -73,6 +73,7 @@ module.exports = (app) => {
// route to run a publish request on the daemon // route to run a publish request on the daemon
app.post('/api/publish', multipartMiddleware, ({ body, files, ip, originalUrl, user }, res) => { app.post('/api/publish', multipartMiddleware, ({ body, files, ip, originalUrl, user }, res) => {
logger.debug('api/publish body:', body); logger.debug('api/publish body:', body);
logger.debug('api/publish body:', files);
let file, fileName, filePath, fileType, name, nsfw, license, title, description, thumbnail, anonymous, skipAuth, channelName, channelPassword; let file, fileName, filePath, fileType, name, nsfw, license, title, description, thumbnail, anonymous, skipAuth, channelName, channelPassword;
// validate that mandatory parts of the request are present // validate that mandatory parts of the request are present
try { try {
@ -84,7 +85,7 @@ module.exports = (app) => {
} }
// validate file, name, license, and nsfw // validate file, name, license, and nsfw
file = files.file; file = files.file;
fileName = file.name; fileName = file.path.substring(file.path.lastIndexOf('/') + 1);
filePath = file.path; filePath = file.path;
fileType = file.type; fileType = file.type;
name = body.name; name = body.name;

View file

@ -6,7 +6,7 @@ const expressHandlebars = require('express-handlebars');
const Handlebars = require('handlebars'); const Handlebars = require('handlebars');
const handlebarsHelpers = require('./helpers/handlebarsHelpers.js'); const handlebarsHelpers = require('./helpers/handlebarsHelpers.js');
const { populateLocalsDotUser, serializeSpeechUser, deserializeSpeechUser } = require('./helpers/authHelpers.js'); const { populateLocalsDotUser, serializeSpeechUser, deserializeSpeechUser } = require('./helpers/authHelpers.js');
const config = require('config'); const config = require('./config/speechConfig.js');
const logger = require('winston'); const logger = require('winston');
const { getDownloadDirectory } = require('./helpers/lbryApi'); const { getDownloadDirectory } = require('./helpers/lbryApi');
const helmet = require('helmet'); const helmet = require('helmet');
@ -17,9 +17,9 @@ const passport = require('passport');
const cookieSession = require('cookie-session'); const cookieSession = require('cookie-session');
// configure logging // configure logging
const logLevel = config.get('Logging.LogLevel'); const logLevel = config.logging.logLevel;
require('./config/loggerConfig.js')(logger, logLevel); require('./config/loggerConfig.js')(logger, logLevel);
require('./config/slackLoggerConfig.js')(logger); require('./config/slackConfig.js')(logger);
// check for global config variables // check for global config variables
require('./helpers/configVarCheck.js')(); require('./helpers/configVarCheck.js')();
@ -42,7 +42,7 @@ app.use((req, res, next) => { // custom logging middleware to log all incoming
// initialize passport // initialize passport
app.use(cookieSession({ app.use(cookieSession({
name : 'session', name : 'session',
keys : [config.get('Session.SessionKey')], keys : [config.session.sessionKey],
maxAge: 24 * 60 * 60 * 1000, // 24 hours maxAge: 24 * 60 * 60 * 1000, // 24 hours
})); }));
app.use(passport.initialize()); app.use(passport.initialize());