Multipart upload directory #243

Merged
bones7242 merged 5 commits from multipart-upload-directory into upload-refactor 2017-11-07 01:05:04 +01:00
19 changed files with 72 additions and 116 deletions

1
.gitignore vendored
View file

@ -1,3 +1,4 @@
node_modules
.idea
config/config.json
config/speechConfig.js

View file

@ -14,13 +14,10 @@ spee.ch is a single-serving site that reads and publishes images and videos to a
* start spee.ch
* clone this repo
* run `npm install`
* to start the server, from your command line run `node speech.js` while passing three environmental variables:
* (1) your lbry wallet address (`LBRY_CLAIM_ADDRESS`),
* (2) your mysql username (`MYSQL_USERNAME`),
* (2) your mysql password (`MYSQL_PASSWORD`),
* (3) the environment to run (`NODE_ENV`).
* i.e. `LBRY_CLAIM_ADDRESS=<your wallet address here> MYSQL_USERNAME=<username here> MYSQL_PASSWORD=<password here> NODE_ENV=development node speech.js`
* e.g. `LBRY_CLAIM_ADDRESS=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX MYSQL_USERNAME="lbry" MYSQL_PASSWORD="xxxxxx" NODE_ENV=development node speech.js`
* create your `speechConfig.js` file
* copy `speechConfig_example.js` and name it `speechConfig.js`
* replace the `null` values in the config file with the appropriate values for your environement
* to start the server, from your command line run `node speech.js`
* To run hot, use `nodemon` instead of `node`
* visit [localhost:3000](http://localhost:3000)

View file

@ -1,15 +0,0 @@
{
"WalletConfig": {
"LbryClaimAddress": "LBRY_CLAIM_ADDRESS"
},
"Database": {
"Username": "MYSQL_USERNAME",
"Password": "MYSQL_PASSWORD"
},
"Logging": {
"SlackWebHook": "SLACK_WEB_HOOK"
},
"Session": {
"SessionKey": "SESSION_KEY"
}
}

View file

@ -1 +0,0 @@
{}

View file

@ -1,23 +0,0 @@
{
"WalletConfig": {
"LbryClaimAddress": null,
"DefaultChannel": null
},
"AnalyticsConfig":{
"GoogleId": null
},
"Database": {
"Database": "lbry",
"Username": null,
"Password": null
},
"Logging": {
"LogLevel": null,
"SlackWebHook": null,
"SlackErrorChannel": null,
"SlackInfoChannel": null
},
"Session": {
"SessionKey": null
}
}

View file

@ -1,13 +0,0 @@
{
"WalletConfig": {
"DefaultChannel": "@speechDev"
},
"AnalyticsConfig":{
"GoogleId": "UA-100747990-1"
},
"Logging": {
"LogLevel": "silly",
"SlackErrorChannel": "#staging_speech-errors",
"SlackInfoChannel": "none"
}
}

View file

@ -1,13 +0,0 @@
{
"WalletConfig": {
"DefaultChannel": "@speech"
},
"AnalyticsConfig":{
"GoogleId": "UA-60403362-3"
},
"Logging": {
"LogLevel": "verbose",
"SlackErrorChannel": "#speech-errors",
"SlackInfoChannel": "#speech-logs"
}
}

View file

@ -1,25 +1,22 @@
const config = require('config');
const SLACK_WEB_HOOK = config.get('Logging.SlackWebHook');
const SLACK_ERROR_CHANNEL = config.get('Logging.SlackErrorChannel');
const SLACK_INFO_CHANNEL = config.get('Logging.SlackInfoChannel');
const config = require('./speechConfig.js');
const winstonSlackWebHook = require('winston-slack-webhook').SlackWebHook;
module.exports = (winston) => {
if (SLACK_WEB_HOOK) {
if (config.logging.slackWebHook) {
// add a transport for errors to slack
winston.add(winstonSlackWebHook, {
name : 'slack-errors-transport',
level : 'error',
webhookUrl: SLACK_WEB_HOOK,
channel : SLACK_ERROR_CHANNEL,
webhookUrl: config.logging.slackWebHook,
channel : config.logging.slackErrorChannel,
username : 'spee.ch',
iconEmoji : ':face_with_head_bandage:',
});
winston.add(winstonSlackWebHook, {
name : 'slack-info-transport',
level : 'info',
webhookUrl: SLACK_WEB_HOOK,
channel : SLACK_INFO_CHANNEL,
webhookUrl: config.logging.slackWebHook,
channel : config.logging.slackInfoChannel,
username : 'spee.ch',
iconEmoji : ':nerd_face:',
});

View file

@ -0,0 +1,22 @@
module.exports = {
wallet: {
lbryClaimAddress: null, // choose an address from your lbry wallet
},
analytics: {
googleId: null, // google id for analytics tracking; leave `null` if not applicable
},
sql: {
database: null, // name of mysql database
username: null, // username for mysql
password: null, // password for mysql
},
logging: {
logLevel : null, // options: silly, debug, verbose, info
slackWebHook : null, // enter a webhook if you wish to push logs to slack; otherwise leave as `null`
slackErrorChannel: null, // enter a slack channel (#example) for errors to be sent to; otherwise leave null
slackInfoChannel : null, // enter a slack channel (#info) for info level logs to be sent to otherwise leave null
},
session: {
sessionKey: null, // enter a secret key to be used for session encryption
},
};

View file

@ -6,13 +6,13 @@ const publishHelpers = require('../helpers/publishHelpers.js');
module.exports = {
publish (publishParams, fileName, fileType) {
return new Promise((resolve, reject) => {
let publishResults = {};
// 1. publish the file
let publishResults, certificateId, channelName;
// publish the file
return lbryApi.publishClaim(publishParams)
// 2. upsert File record (update is in case the claim has been published before by this daemon)
.then(tx => {
logger.info(`Successfully published ${fileName}`, tx);
publishResults = tx;
// get the channel information
if (publishParams.channel_name) {
logger.debug(`this claim was published in channel: ${publishParams.channel_name}`);
return db.Channel.findOne({where: {channelName: publishParams.channel_name}});
@ -22,13 +22,17 @@ module.exports = {
}
})
.then(channel => {
let certificateId = null;
let channelName = null;
// set channel information
certificateId = null;
channelName = null;
if (channel) {
certificateId = channel.channelClaimId;
channelName = channel.channelName;
}
logger.debug(`certificateId: ${certificateId}`);
})
.then(() => {
// create the File record
const fileRecord = {
name : publishParams.name,
claimId : publishResults.claim_id,
@ -42,6 +46,7 @@ module.exports = {
fileType,
nsfw : publishParams.metadata.nsfw,
};
// create the Claim record
const claimRecord = {
name : publishParams.name,
claimId : publishResults.claim_id,
@ -57,11 +62,12 @@ module.exports = {
certificateId,
channelName,
};
// upsert criteria
const upsertCriteria = {
name : publishParams.name,
claimId: publishResults.claim_id,
};
// create the records
// upsert the records
return Promise.all([db.upsert(db.File, fileRecord, upsertCriteria, 'File'), db.upsert(db.Claim, claimRecord, upsertCriteria, 'Claim')]);
})
.then(([file, claim]) => {

View file

@ -1,8 +1,8 @@
const logger = require('winston');
const ua = require('universal-analytics');
const config = require('config');
const config = require('../config/speechConfig.js');
const db = require('../models');
const googleApiKey = config.get('AnalyticsConfig.GoogleId');
const googleApiKey = config.analytics.googleId;
module.exports = {
postToStats (action, url, ipAddress, name, claimId, result) {

View file

@ -1,15 +1,13 @@
const config = require('config');
const config = require('../config/speechConfig.js');
const logger = require('winston');
const fs = require('fs');
module.exports = function () {
// get the config file
const defaultConfigFile = JSON.parse(fs.readFileSync('./config/default.json'));
for (let configCategoryKey in defaultConfigFile) {
if (defaultConfigFile.hasOwnProperty(configCategoryKey)) {
for (let configCategoryKey in config) {
if (config.hasOwnProperty(configCategoryKey)) {
// get the final variables for each config category
const configVariables = config.get(configCategoryKey);
const configVariables = config[configCategoryKey];
for (let configVarKey in configVariables) {
if (configVariables.hasOwnProperty(configVarKey)) {
// print each variable

View file

@ -1,10 +1,10 @@
const Handlebars = require('handlebars');
const config = require('config');
const config = require('../config/speechConfig.js');
module.exports = {
// define any extra helpers you may need
googleAnalytics () {
const googleApiKey = config.get('AnalyticsConfig.GoogleId');
const googleApiKey = config.analytics.googleId;
return new Handlebars.SafeString(
`<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){

View file

@ -1,7 +1,7 @@
const logger = require('winston');
const fs = require('fs');
const db = require('../models');
const config = require('config');
const config = require('../config/speechConfig.js');
module.exports = {
validateApiPublishRequest (body, files) {
@ -110,7 +110,7 @@ module.exports = {
license,
nsfw,
},
claim_address: config.get('WalletConfig.LbryClaimAddress'),
claim_address: config.wallet.lbryClaimAddress,
};
// add thumbnail to channel if video
if (thumbnail !== null) {
@ -137,7 +137,7 @@ module.exports = {
db.File.findAll({ where: { name } })
.then(result => {
if (result.length >= 1) {
const claimAddress = config.get('WalletConfig.LbryClaimAddress');
const claimAddress = config.wallet.lbryClaimAddress;
// filter out any results that were not published from spee.ch's wallet address
const filteredResult = result.filter((claim) => {
return (claim.address === claimAddress);

View file

@ -2,13 +2,13 @@ const fs = require('fs');
const path = require('path');
const Sequelize = require('sequelize');
const basename = path.basename(module.filename);
const config = require('config');
const config = require('../config/speechConfig.js');
const db = {};
const logger = require('winston');
const database = config.get('Database.Database');
const username = config.get('Database.Username');
const password = config.get('Database.Password');
const database = config.sql.database;
const username = config.sql.username;
const password = config.sql.password;
const sequelize = new Sequelize(database, username, password, {
host : 'localhost',
@ -66,7 +66,7 @@ db.upsert = (Model, values, condition, tableName) => {
}
})
.catch(function (error) {
logger.error('Sequelize findOne error', error);
logger.error(`${tableName}.upsert error`, error);
});
};

View file

@ -34,7 +34,6 @@
"cookie-session": "^2.0.0-beta.3",
"express": "^4.15.2",
"express-handlebars": "^3.0.0",
"express-session": "^1.15.5",
"form-data": "^2.3.1",
"helmet": "^3.8.1",
"mysql2": "^1.3.5",

View file

@ -175,7 +175,7 @@ var publishFileFunctions = {
},
showUploadProgressMessage: function (percentage){
this.updatePublishStatus('<p>File is loading to server</p>');
this.updateUploadPercent('<p class="blue">' + percentage + '</p>');
this.updateUploadPercent('<p class="blue">' + percentage + '% </p>');
},
showFilePublishUpdate: function (msg) {
this.updatePublishStatus('<p>' + msg + '</p>');

View file

@ -1,6 +1,6 @@
const logger = require('winston');
const multipart = require('connect-multiparty');
const multipartMiddleware = multipart();
const multipartMiddleware = multipart({uploadDir: '/home/lbry/test/'});
const db = require('../models');
const { publish } = require('../controllers/publishController.js');
const { getClaimList, resolveUri } = require('../helpers/lbryApi.js');
@ -73,6 +73,7 @@ module.exports = (app) => {
// route to run a publish request on the daemon
app.post('/api/publish', multipartMiddleware, ({ body, files, ip, originalUrl, user }, res) => {
logger.debug('api/publish body:', body);
logger.debug('api/publish body:', files);
let file, fileName, filePath, fileType, name, nsfw, license, title, description, thumbnail, anonymous, skipAuth, channelName, channelPassword;
// validate that mandatory parts of the request are present
try {
@ -84,7 +85,7 @@ module.exports = (app) => {
}
// validate file, name, license, and nsfw
file = files.file;
fileName = file.name;
fileName = file.path.substring(file.path.lastIndexOf('/') + 1);
filePath = file.path;
fileType = file.type;
name = body.name;

View file

@ -6,7 +6,7 @@ const expressHandlebars = require('express-handlebars');
const Handlebars = require('handlebars');
const handlebarsHelpers = require('./helpers/handlebarsHelpers.js');
const { populateLocalsDotUser, serializeSpeechUser, deserializeSpeechUser } = require('./helpers/authHelpers.js');
const config = require('config');
const config = require('./config/speechConfig.js');
const logger = require('winston');
const { getDownloadDirectory } = require('./helpers/lbryApi');
const helmet = require('helmet');
@ -17,9 +17,9 @@ const passport = require('passport');
const cookieSession = require('cookie-session');
// configure logging
const logLevel = config.get('Logging.LogLevel');
const logLevel = config.logging.logLevel;
require('./config/loggerConfig.js')(logger, logLevel);
require('./config/slackLoggerConfig.js')(logger);
require('./config/slackConfig.js')(logger);
// check for global config variables
require('./helpers/configVarCheck.js')();
@ -42,7 +42,7 @@ app.use((req, res, next) => { // custom logging middleware to log all incoming
// initialize passport
app.use(cookieSession({
name : 'session',
keys : [config.get('Session.SessionKey')],
keys : [config.session.sessionKey],
maxAge: 24 * 60 * 60 * 1000, // 24 hours
}));
app.use(passport.initialize());