aiden
/
YoutubeDL-Material
mirror of https://github.com/Tzahi12345/YoutubeDL-Material
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improved DB tests, now both local and remote DB can be tested easily

Browse Source
pull/809/head
Tzahi12345 3 years ago
parent f20a31ed0f
commit c724a8019a
4 changed files with 169 additions and 131 deletions
Show Stats Download Patch File Download Diff File

5
.vscode/settings.json vendored
Unescape Escape View File

@ -1,5 +1,8 @@
{ {
"mochaExplorer.files": "backend/test/**/*.js", "mochaExplorer.files": "backend/test/**/*.js",
"mochaExplorer.cwd": "backend", "mochaExplorer.cwd": "backend",
"mochaExplorer.globImplementation": "vscode" "mochaExplorer.globImplementation": "vscode",
"mochaExplorer.env": {
"YTDL_MODE": "debug"
}
} }

8
backend/db.js
Unescape Escape View File

@ -2,6 +2,7 @@ var fs = require('fs-extra')
var path = require('path') var path = require('path')
const { MongoClient } = require("mongodb"); const { MongoClient } = require("mongodb");
const { uuid } = require('uuidv4'); const { uuid } = require('uuidv4');
const _ = require('lodash');
const config_api = require('./config'); const config_api = require('./config');
var utils = require('./utils') var utils = require('./utils')
@ -152,6 +153,7 @@ exports._connectToDB = async (custom_connection_string = null) => {
await database.collection(table).createIndex(text_search); await database.collection(table).createIndex(text_search);
} }
}); });
using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
return true; return true;
} catch(err) { } catch(err) {
logger.error(err); logger.error(err);
@ -578,7 +580,6 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => {
exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => { exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
// local db override // local db override
if (using_local_db) { if (using_local_db) {
if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
local_db.get(table).push(doc).write(); local_db.get(table).push(doc).write();
return true; return true;
} }
@ -1141,3 +1142,8 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
}); });
return return_val; return return_val;
} }
// should only be used for tests
exports.setLocalDBMode = (mode) => {
using_local_db = mode;
}

1
backend/package.json
Unescape Escape View File

@ -32,6 +32,7 @@
"fs-extra": "^9.0.0", "fs-extra": "^9.0.0",
"gotify": "^1.1.0", "gotify": "^1.1.0",
"jsonwebtoken": "^8.5.1", "jsonwebtoken": "^8.5.1",
"lodash": "^4.17.21",
"lowdb": "^1.0.0", "lowdb": "^1.0.0",
"md5": "^2.2.1", "md5": "^2.2.1",
"merge-files": "^0.1.2", "merge-files": "^0.1.2",

286
backend/test/tests.js
Unescape Escape View File

@ -104,151 +104,179 @@ describe('Database', async function() {
}); });
describe('Basic functions', async function() { describe('Basic functions', async function() {
beforeEach(async function() {
await db_api.connectToDB(); // test both local_db and remote_db
await db_api.removeAllRecords('test'); const local_db_modes = [false, true];
});
it('Add and read record', async function() { for (const local_db_mode of local_db_modes) {
this.timeout(120000); let use_local_db = local_db_mode;
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined}); describe(`Use local DB - ${use_local_db}`, async function() {
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null}); beforeEach(async function() {
assert(added_record['test_add'] === 'test'); if (!use_local_db) {
await db_api.removeRecord('test', {test_add: 'test'}); this.timeout(120000);
}); await db_api.connectToDB(0);
}
it('Find duplicates by key', async function() { await db_api.removeAllRecords('test');
const test_duplicates = [ });
{ it('Add and read record', async function() {
test: 'testing', this.timeout(120000);
key: '1' await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
}, const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
{ assert(added_record['test_add'] === 'test');
test: 'testing', await db_api.removeRecord('test', {test_add: 'test'});
key: '2' });
}, it('Add and read record - Nested property', async function() {
{ this.timeout(120000);
test: 'testing_missing', await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
key: '3' const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
}, const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
{ assert(added_record['test_add'] === 'test');
test: 'testing', assert(!not_added_record);
key: '4' await db_api.removeRecord('test', {test_add: 'test'});
} });
]; it('Replace filter', async function() {
await db_api.insertRecordsIntoTable('test', test_duplicates); this.timeout(120000);
const duplicates = await db_api.findDuplicatesByKey('test', 'test'); await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
console.log(duplicates); await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
}); const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
assert(count === 1);
it('Update record', async function() { await db_api.removeRecord('test', {test_replace_filter: 'test'});
await db_api.insertRecordIntoTable('test', {test_update: 'test'}); });
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true}); it('Find duplicates by key', async function() {
const updated_record = await db_api.getRecord('test', {test_update: 'test'}); const test_duplicates = [
assert(updated_record['added_field']); {
await db_api.removeRecord('test', {test_update: 'test'}); test: 'testing',
}); key: '1'
},
it('Remove record', async function() { {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'}); test: 'testing',
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'}); key: '2'
assert(delete_succeeded); },
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'}); {
assert(!deleted_record); test: 'testing_missing',
}); key: '3'
},
it('Push to record array', async function() { {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []}); test: 'testing',
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item'); key: '4'
const record = await db_api.getRecord('test', {test: 'test'}); }
assert(record); ];
assert(record['test_array'].length === 1); await db_api.insertRecordsIntoTable('test', test_duplicates);
}); const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
it('Pull from record array', async function() { });
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
it('Bulk add', async function() { it('Update record', async function() {
this.timeout(120000); await db_api.insertRecordIntoTable('test', {test_update: 'test'});
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000 await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const test_records = []; const updated_record = await db_api.getRecord('test', {test_update: 'test'});
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { assert(updated_record['added_field']);
test_records.push({ await db_api.removeRecord('test', {test_update: 'test'});
uid: uuid()
}); });
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test'); it('Remove record', async function() {
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD); await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
}); const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Bulk update', async function() { it('Push to record array', async function() {
// bulk add records await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000 await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const test_records = []; const record = await db_api.getRecord('test', {test: 'test'});
const update_obj = {}; assert(record);
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { assert(record['test_array'].length === 1);
const test_uid = uuid();
test_records.push({
uid: test_uid
}); });
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added it('Pull from record array', async function() {
const received_records = await db_api.getRecords('test'); await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD); await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj); it('Bulk add', async function() {
assert(success); this.timeout(120000);
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
const received_updated_records = await db_api.getRecords('test'); it('Bulk update', async function() {
for (let i = 0; i < received_updated_records.length; i++) { // bulk add records
success &= received_updated_records[i]['added_field']; const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
} const test_records = [];
assert(success); const update_obj = {};
}); for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
assert(success);
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Stats', async function() { it('Stats', async function() {
const stats = await db_api.getDBStats(); const stats = await db_api.getDBStats();
assert(stats); assert(stats);
}); });
it('Query speed', async function() { it('Query speed', async function() {
this.timeout(120000); this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000 const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = []; const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943'; let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid(); const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid; if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632}); test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
} }
const insert_start = Date.now(); const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records); let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now(); const insert_end = Date.now();
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`); console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
const query_start = Date.now(); const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid}); const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now(); const query_end = Date.now();
console.log(random_record) console.log(random_record)
console.log(`Query time: ${(query_end - query_start)/1000}s`); console.log(`Query time: ${(query_end - query_start)/1000}s`);
success = !!random_record; success = !!random_record;
assert(success); assert(success);
}); });
});
}
}); });
describe('Local DB Filters', async function() { describe('Local DB Filters', async function() {

Write Preview
Loading…
Cancel
Save