diff --git a/controller/websocket.js b/controller/websocket.js index a793026..6d8fa23 100644 --- a/controller/websocket.js +++ b/controller/websocket.js @@ -117,6 +117,10 @@ exports.playlist = async (ws, req) => { async function startDownloading() { const playlist = await metadata.getPlaylistVideos(playlistId) for (video of playlist.relatedStreams) { + if (ws.readyState !== ws.OPEN) { + return logger.info({ message: `Stopped downloading ${playlistId}, websocket is closed` }) + } + const id = video.url.match(/[?&]v=([^&]+)/)[1] const already = await prisma.videos.findFirst({ @@ -159,16 +163,21 @@ exports.playlist = async (ws, req) => { } else { const file = fs.readdirSync("./videos").find(f => f.includes(id)) if (file) { - fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`) - ws.send(`DATA - Downloaded ${video.title}`) - ws.send(`DATA - Uploading ${video.title}`) + try { + fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`) + ws.send(`DATA - Downloaded ${video.title}`) + ws.send(`DATA - Uploading ${video.title}`) - const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`) - ws.send(`DATA - Uploaded ${video.title}`) - fs.unlinkSync(`./videos/${id}.webm`) + const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`) + ws.send(`DATA - Uploaded ${video.title}`) + fs.unlinkSync(`./videos/${id}.webm`) - await websocket.createDatabaseVideo(id, videoUrl, playlistId) - ws.send(`DATA - Created video page for ${video.title}`) + await websocket.createDatabaseVideo(id, videoUrl, playlistId) + ws.send(`DATA - Created video page for ${video.title}`) + } catch (e) { + ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`) + logger.error(e) + } } else { ws.send(`DATA - Failed to find file for ${video.title}. Going to next video in the playlist`) } @@ -212,6 +221,10 @@ exports.channel = async (ws, req) => { const videos = await metadata.getChannelVideos(channelId) for (video of videos) { + if (ws.readyState !== ws.OPEN) { + return logger.info({ message: `Stopped downloading ${channelId}, websocket is closed` }) + } + const id = video.url.match(/[?&]v=([^&]+)/)[1] const already = await prisma.videos.findFirst({ @@ -246,16 +259,21 @@ exports.channel = async (ws, req) => { } else { const file = fs.readdirSync("./videos").find(f => f.includes(id)) if (file) { - fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`) - ws.send(`DATA - Downloaded ${video.title}`) - ws.send(`DATA - Uploading ${video.title}`) + try { + fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`) + ws.send(`DATA - Downloaded ${video.title}`) + ws.send(`DATA - Uploading ${video.title}`) - const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`) - ws.send(`DATA - Uploaded ${video.title}`) - fs.unlinkSync(`./videos/${id}.webm`) + const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`) + ws.send(`DATA - Uploaded ${video.title}`) + fs.unlinkSync(`./videos/${id}.webm`) - await websocket.createDatabaseVideo(id, videoUrl) - ws.send(`DATA - Created video page for ${video.title}`) + await websocket.createDatabaseVideo(id, videoUrl) + ws.send(`DATA - Created video page for ${video.title}`) + } catch (e) { + ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`) + logger.error(e) + } } else { ws.send(`DATA - Failed to find file for ${video.title}. Going to next video`) } diff --git a/index.js b/index.js index 8d6f635..809c958 100644 --- a/index.js +++ b/index.js @@ -34,7 +34,7 @@ app.ws('/savechannel', websocketController.channel) app.get('/autodownload', websocketController.addAutodownload) process.on('uncaughtException', err => { - logger.info({ message: `Error: ${err.message}` }) + logger.error(err) }) app.listen(1337, () => { diff --git a/package.json b/package.json index dad4e73..41ab246 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,8 @@ "main": "index.js", "license": "AGPL-3.0", "dependencies": { + "@logtail/node": "^0.4.0", + "@logtail/winston": "^0.4.1", "@prisma/client": "4.9.0", "aws-sdk": "2.1128.0", "child_process": "^1.0.2", diff --git a/utils/logger.js b/utils/logger.js index 377fbdf..8f7b77b 100644 --- a/utils/logger.js +++ b/utils/logger.js @@ -1,4 +1,9 @@ const winston = require('winston') +const { Logtail } = require("@logtail/node") +const { LogtailTransport } = require("@logtail/winston") + +const logtail = new Logtail("YFQdKmZgGvxPpusqxCSxsj2b") + const logger = winston.createLogger({ format: winston.format.json(), transports: [ @@ -13,6 +18,9 @@ const logger = winston.createLogger({ winston.format.timestamp({format: 'MMM-DD-YYYY HH:mm:ss'}), winston.format.printf(info => `${[info.timestamp]}: ${info.message}`), )}), + new LogtailTransport(logtail, { + level: 'error' + }) ], }); diff --git a/yarn.lock b/yarn.lock index 9572966..b4545a5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -21,6 +21,55 @@ resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== +"@logtail/core@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@logtail/core/-/core-0.4.0.tgz#a8e510756bf9fd1b2cc6c901225eb03685919ca8" + integrity sha512-xItEU5vY2LvcJyAVkxKKZ3UvG3CsK2pMvhuLDntlC+US1LlNcfQmVzvInN5j5iXl0kjNO6n+0EY2YyhhOgjWCQ== + dependencies: + "@logtail/tools" "^0.4.0" + "@logtail/types" "^0.4.0" + +"@logtail/node@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@logtail/node/-/node-0.4.0.tgz#71488911a6dc4bf218871af80e3788cd727f8474" + integrity sha512-0/1to4HLf95FrDlZoFLz3A0Fz/B55G82syY22Yl9N4mjC2wW6t5Lo9d4q/u/VmHBHW3PBbytbmFEoHx0H5Sarg== + dependencies: + "@logtail/core" "^0.4.0" + "@logtail/types" "^0.4.0" + "@msgpack/msgpack" "^2.5.1" + "@types/stack-trace" "^0.0.29" + cross-fetch "^3.0.4" + minimatch "^3.0.4" + stack-trace "^0.0.10" + +"@logtail/tools@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@logtail/tools/-/tools-0.4.0.tgz#ff1bed193a005b12007df510f6ce294b8f518207" + integrity sha512-7lSKdJTq7NMUvriMhkf6cTxY9QlXi+YLLuHxbSOKI56HpJN8qJGPVcmOdzlbk8pJofe9RnME8hBMLTHAD3bmsw== + dependencies: + "@logtail/types" "^0.4.0" + +"@logtail/types@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@logtail/types/-/types-0.4.0.tgz#39c28e08f805657c70107071dc166866153563a8" + integrity sha512-2CR3w7Xf5rKWbUlgiaBKeVrqWuWnVBz0Ymycw/gYwupVI0fb6Ameid9fHfeM5LI/gbIO3ZTIMADlA0FvFbuXMQ== + dependencies: + js "^0.1.0" + +"@logtail/winston@^0.4.1": + version "0.4.1" + resolved "https://registry.yarnpkg.com/@logtail/winston/-/winston-0.4.1.tgz#bbdb60db0ae1e013a1e9678ba25160eb2554fecf" + integrity sha512-YsMXzhcpGHDd3ijyqnDD+2qraJAiar/mVjR7HP8NYQsQMk9dQTIHbQzXkBUwAfVOsKSzNbfzeWTnft8sQBYxYQ== + dependencies: + "@logtail/node" "^0.4.0" + "@logtail/types" "^0.4.0" + winston-transport "^4.3.0" + +"@msgpack/msgpack@^2.5.1": + version "2.8.0" + resolved "https://registry.yarnpkg.com/@msgpack/msgpack/-/msgpack-2.8.0.tgz#4210deb771ee3912964f14a15ddfb5ff877e70b9" + integrity sha512-h9u4u/jiIRKbq25PM+zymTyW6bhTzELvOoUd+AvYriWOAKpLGnIamaET3pnHYoI5iYphAHBI4ayx0MehR+VVPQ== + "@prisma/client@4.9.0": version "4.9.0" resolved "https://registry.yarnpkg.com/@prisma/client/-/client-4.9.0.tgz#4a4068f3540732ea5723c008d49ed684d20f9340" @@ -50,6 +99,11 @@ dependencies: "@types/trusted-types" "*" +"@types/stack-trace@^0.0.29": + version "0.0.29" + resolved "https://registry.yarnpkg.com/@types/stack-trace/-/stack-trace-0.0.29.tgz#eb7a7c60098edb35630ed900742a5ecb20cfcb4d" + integrity sha512-TgfOX+mGY/NyNxJLIbDWrO9DjGoVSW9+aB8H2yy1fy32jsvxijhmyJI9fDFgvz3YP4lvJaq9DzdR/M1bOgVc9g== + "@types/triple-beam@^1.3.2": version "1.3.2" resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.2.tgz#38ecb64f01aa0d02b7c8f4222d7c38af6316fef8" @@ -128,6 +182,11 @@ aws-sdk@2.1128.0: uuid "3.3.2" xml2js "0.4.19" +balanced-match@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== + base64-js@^1.0.2: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" @@ -151,6 +210,14 @@ body-parser@1.20.1: type-is "~1.6.18" unpipe "1.0.0" +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + buffer@4.9.2: version "4.9.2" resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" @@ -231,6 +298,18 @@ combined-stream@^1.0.8: dependencies: delayed-stream "~1.0.0" +commander@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-1.1.1.tgz#50d1651868ae60eccff0a2d9f34595376bc6b041" + integrity sha512-71Rod2AhcH3JhkBikVpNd0pA+fWsmAaVoti6OR38T76chA7vE3pSerS0Jor4wDw+tOueD2zLVvFOw5H0Rcj7rA== + dependencies: + keypress "0.1.x" + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== + content-disposition@0.5.4: version "0.5.4" resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" @@ -261,6 +340,13 @@ cors@^2.8.5: object-assign "^4" vary "^1" +cross-fetch@^3.0.4: + version "3.1.8" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82" + integrity sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg== + dependencies: + node-fetch "^2.6.12" + cssom@^0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.5.0.tgz#d254fa92cd8b6fbd83811b9fbaed34663cc17c36" @@ -645,6 +731,13 @@ jmespath@0.16.0: resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076" integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw== +js@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/js/-/js-0.1.0.tgz#e1d0afd55ea39c2b28da304e8143eaf2c133f366" + integrity sha512-ZBbGYOpact8QAH9RprFWL4RAESYwbDodxiuDjOnzwzzk9pBzKycoifGuUrHHcDixE/eLMKPHRaXenTgu1qXBqA== + dependencies: + commander "~1.1.1" + jsdom@^21.1.0: version "21.1.0" resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-21.1.0.tgz#d56ba4a84ed478260d83bd53dc181775f2d8e6ef" @@ -677,6 +770,11 @@ jsdom@^21.1.0: ws "^8.11.0" xml-name-validator "^4.0.0" +keypress@0.1.x: + version "0.1.0" + resolved "https://registry.yarnpkg.com/keypress/-/keypress-0.1.0.tgz#4a3188d4291b66b4f65edb99f806aa9ae293592a" + integrity sha512-x0yf9PL/nx9Nw9oLL8ZVErFAk85/lslwEP7Vz7s5SI1ODXZIgit3C5qyWjw4DxOuO/3Hb4866SQh28a1V1d+WA== + kuler@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" @@ -744,6 +842,13 @@ mime@1.6.0: resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== +minimatch@^3.0.4: + version "3.1.2" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== + dependencies: + brace-expansion "^1.1.7" + ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -771,6 +876,13 @@ node-fetch@2: dependencies: whatwg-url "^5.0.0" +node-fetch@^2.6.12: + version "2.6.12" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.12.tgz#02eb8e22074018e3d5a83016649d04df0e348fba" + integrity sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g== + dependencies: + whatwg-url "^5.0.0" + nwsapi@^2.2.2: version "2.2.2" resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.2.tgz#e5418863e7905df67d51ec95938d67bf801f0bb0" @@ -1009,7 +1121,7 @@ source-map@~0.6.1: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== -stack-trace@0.0.x: +stack-trace@0.0.x, stack-trace@^0.0.10: version "0.0.10" resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" integrity sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg== @@ -1179,7 +1291,7 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" -winston-transport@^4.5.0: +winston-transport@^4.3.0, winston-transport@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.5.0.tgz#6e7b0dd04d393171ed5e4e4905db265f7ab384fa" integrity sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==