logtail, catching errors while downloading playlists and channels, stop when websocket is closed

controller/websocket.js

@@ -117,6 +117,10 @@ exports.playlist = async (ws, req) => {
     async function startDownloading() {
         const playlist = await metadata.getPlaylistVideos(playlistId)
         for (video of playlist.relatedStreams) {
+            if (ws.readyState !== ws.OPEN) {
+                return logger.info({ message: `Stopped downloading ${playlistId}, websocket is closed` })
+            }
+
             const id = video.url.match(/[?&]v=([^&]+)/)[1]
 
             const already = await prisma.videos.findFirst({
@@ -159,16 +163,21 @@ exports.playlist = async (ws, req) => {
             } else {
                 const file = fs.readdirSync("./videos").find(f => f.includes(id))
                 if (file) {
-                    fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
+                    try {
-                    ws.send(`DATA - Downloaded ${video.title}`)
+                        fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
-                    ws.send(`DATA - Uploading ${video.title}`)
+                        ws.send(`DATA - Downloaded ${video.title}`)
+                        ws.send(`DATA - Uploading ${video.title}`)
 
-                    const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
+                        const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
-                    ws.send(`DATA - Uploaded ${video.title}`)
+                        ws.send(`DATA - Uploaded ${video.title}`)
-                    fs.unlinkSync(`./videos/${id}.webm`)
+                        fs.unlinkSync(`./videos/${id}.webm`)
 
-                    await websocket.createDatabaseVideo(id, videoUrl, playlistId)
+                        await websocket.createDatabaseVideo(id, videoUrl, playlistId)
-                    ws.send(`DATA - Created video page for ${video.title}`)
+                        ws.send(`DATA - Created video page for ${video.title}`)
+                    } catch (e) {
+                        ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`)
+                        logger.error(e)
+                    }
                 } else {
                     ws.send(`DATA - Failed to find file for ${video.title}. Going to next video in the playlist`)
                 }
@@ -212,6 +221,10 @@ exports.channel = async (ws, req) => {
         const videos = await metadata.getChannelVideos(channelId)
 
         for (video of videos) {
+            if (ws.readyState !== ws.OPEN) {
+                return logger.info({ message: `Stopped downloading ${channelId}, websocket is closed` })
+            }
+
             const id = video.url.match(/[?&]v=([^&]+)/)[1]
 
             const already = await prisma.videos.findFirst({
@@ -246,16 +259,21 @@ exports.channel = async (ws, req) => {
             } else {
                 const file = fs.readdirSync("./videos").find(f => f.includes(id))
                 if (file) {
-                    fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
+                    try {
-                    ws.send(`DATA - Downloaded ${video.title}`)
+                        fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
-                    ws.send(`DATA - Uploading ${video.title}`)
+                        ws.send(`DATA - Downloaded ${video.title}`)
+                        ws.send(`DATA - Uploading ${video.title}`)
 
-                    const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
+                        const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
-                    ws.send(`DATA - Uploaded ${video.title}`)
+                        ws.send(`DATA - Uploaded ${video.title}`)
-                    fs.unlinkSync(`./videos/${id}.webm`)
+                        fs.unlinkSync(`./videos/${id}.webm`)
 
-                    await websocket.createDatabaseVideo(id, videoUrl)
+                        await websocket.createDatabaseVideo(id, videoUrl)
-                    ws.send(`DATA - Created video page for ${video.title}`)
+                        ws.send(`DATA - Created video page for ${video.title}`)
+                    } catch (e) {
+                        ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`)
+                        logger.error(e)
+                    }
                 } else {
                     ws.send(`DATA - Failed to find file for ${video.title}. Going to next video`)
                 }

index.js

@@ -34,7 +34,7 @@ app.ws('/savechannel', websocketController.channel)
 app.get('/autodownload', websocketController.addAutodownload)
 
 process.on('uncaughtException', err => {
-  logger.info({ message: `Error: ${err.message}` })
+  logger.error(err)
 })
 
 app.listen(1337, () => {

package.json

@@ -4,6 +4,8 @@
   "main": "index.js",
   "license": "AGPL-3.0",
   "dependencies": {
+    "@logtail/node": "^0.4.0",
+    "@logtail/winston": "^0.4.1",
     "@prisma/client": "4.9.0",
     "aws-sdk": "2.1128.0",
     "child_process": "^1.0.2",

utils/logger.js

@@ -1,4 +1,9 @@
 const winston = require('winston')
+const { Logtail } = require("@logtail/node")
+const { LogtailTransport } = require("@logtail/winston")
+
+const logtail = new Logtail("YFQdKmZgGvxPpusqxCSxsj2b")
+
 const logger = winston.createLogger({
     format: winston.format.json(),
     transports: [
@@ -13,6 +18,9 @@ const logger = winston.createLogger({
                 winston.format.timestamp({format: 'MMM-DD-YYYY HH:mm:ss'}),
                 winston.format.printf(info => `${[info.timestamp]}: ${info.message}`),
             )}),
+        new LogtailTransport(logtail, {
+            level: 'error'
+        })
     ],
 });
 

yarn.lock

@@ -21,6 +21,55 @@
   resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
   integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==
 
+"@logtail/core@^0.4.0":
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/@logtail/core/-/core-0.4.0.tgz#a8e510756bf9fd1b2cc6c901225eb03685919ca8"
+  integrity sha512-xItEU5vY2LvcJyAVkxKKZ3UvG3CsK2pMvhuLDntlC+US1LlNcfQmVzvInN5j5iXl0kjNO6n+0EY2YyhhOgjWCQ==
+  dependencies:
+    "@logtail/tools" "^0.4.0"
+    "@logtail/types" "^0.4.0"
+
+"@logtail/node@^0.4.0":
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/@logtail/node/-/node-0.4.0.tgz#71488911a6dc4bf218871af80e3788cd727f8474"
+  integrity sha512-0/1to4HLf95FrDlZoFLz3A0Fz/B55G82syY22Yl9N4mjC2wW6t5Lo9d4q/u/VmHBHW3PBbytbmFEoHx0H5Sarg==
+  dependencies:
+    "@logtail/core" "^0.4.0"
+    "@logtail/types" "^0.4.0"
+    "@msgpack/msgpack" "^2.5.1"
+    "@types/stack-trace" "^0.0.29"
+    cross-fetch "^3.0.4"
+    minimatch "^3.0.4"
+    stack-trace "^0.0.10"
+
+"@logtail/tools@^0.4.0":
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/@logtail/tools/-/tools-0.4.0.tgz#ff1bed193a005b12007df510f6ce294b8f518207"
+  integrity sha512-7lSKdJTq7NMUvriMhkf6cTxY9QlXi+YLLuHxbSOKI56HpJN8qJGPVcmOdzlbk8pJofe9RnME8hBMLTHAD3bmsw==
+  dependencies:
+    "@logtail/types" "^0.4.0"
+
+"@logtail/types@^0.4.0":
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/@logtail/types/-/types-0.4.0.tgz#39c28e08f805657c70107071dc166866153563a8"
+  integrity sha512-2CR3w7Xf5rKWbUlgiaBKeVrqWuWnVBz0Ymycw/gYwupVI0fb6Ameid9fHfeM5LI/gbIO3ZTIMADlA0FvFbuXMQ==
+  dependencies:
+    js "^0.1.0"
+
+"@logtail/winston@^0.4.1":
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/@logtail/winston/-/winston-0.4.1.tgz#bbdb60db0ae1e013a1e9678ba25160eb2554fecf"
+  integrity sha512-YsMXzhcpGHDd3ijyqnDD+2qraJAiar/mVjR7HP8NYQsQMk9dQTIHbQzXkBUwAfVOsKSzNbfzeWTnft8sQBYxYQ==
+  dependencies:
+    "@logtail/node" "^0.4.0"
+    "@logtail/types" "^0.4.0"
+    winston-transport "^4.3.0"
+
+"@msgpack/msgpack@^2.5.1":
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/@msgpack/msgpack/-/msgpack-2.8.0.tgz#4210deb771ee3912964f14a15ddfb5ff877e70b9"
+  integrity sha512-h9u4u/jiIRKbq25PM+zymTyW6bhTzELvOoUd+AvYriWOAKpLGnIamaET3pnHYoI5iYphAHBI4ayx0MehR+VVPQ==
+
 "@prisma/client@4.9.0":
   version "4.9.0"
   resolved "https://registry.yarnpkg.com/@prisma/client/-/client-4.9.0.tgz#4a4068f3540732ea5723c008d49ed684d20f9340"
@@ -50,6 +99,11 @@
   dependencies:
     "@types/trusted-types" "*"
 
+"@types/stack-trace@^0.0.29":
+  version "0.0.29"
+  resolved "https://registry.yarnpkg.com/@types/stack-trace/-/stack-trace-0.0.29.tgz#eb7a7c60098edb35630ed900742a5ecb20cfcb4d"
+  integrity sha512-TgfOX+mGY/NyNxJLIbDWrO9DjGoVSW9+aB8H2yy1fy32jsvxijhmyJI9fDFgvz3YP4lvJaq9DzdR/M1bOgVc9g==
+
 "@types/triple-beam@^1.3.2":
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.2.tgz#38ecb64f01aa0d02b7c8f4222d7c38af6316fef8"
@@ -128,6 +182,11 @@ aws-sdk@2.1128.0:
     uuid "3.3.2"
     xml2js "0.4.19"
 
+balanced-match@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
+  integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
+
 base64-js@^1.0.2:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
@@ -151,6 +210,14 @@ body-parser@1.20.1:
     type-is "~1.6.18"
     unpipe "1.0.0"
 
+brace-expansion@^1.1.7:
+  version "1.1.11"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
+  integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==
+  dependencies:
+    balanced-match "^1.0.0"
+    concat-map "0.0.1"
+
 buffer@4.9.2:
   version "4.9.2"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8"
@@ -231,6 +298,18 @@ combined-stream@^1.0.8:
   dependencies:
     delayed-stream "~1.0.0"
 
+commander@~1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-1.1.1.tgz#50d1651868ae60eccff0a2d9f34595376bc6b041"
+  integrity sha512-71Rod2AhcH3JhkBikVpNd0pA+fWsmAaVoti6OR38T76chA7vE3pSerS0Jor4wDw+tOueD2zLVvFOw5H0Rcj7rA==
+  dependencies:
+    keypress "0.1.x"
+
+concat-map@0.0.1:
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
+  integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==
+
 content-disposition@0.5.4:
   version "0.5.4"
   resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe"
@@ -261,6 +340,13 @@ cors@^2.8.5:
     object-assign "^4"
     vary "^1"
 
+cross-fetch@^3.0.4:
+  version "3.1.8"
+  resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82"
+  integrity sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==
+  dependencies:
+    node-fetch "^2.6.12"
+
 cssom@^0.5.0:
   version "0.5.0"
   resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.5.0.tgz#d254fa92cd8b6fbd83811b9fbaed34663cc17c36"
@@ -645,6 +731,13 @@ jmespath@0.16.0:
   resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076"
   integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==
 
+js@^0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/js/-/js-0.1.0.tgz#e1d0afd55ea39c2b28da304e8143eaf2c133f366"
+  integrity sha512-ZBbGYOpact8QAH9RprFWL4RAESYwbDodxiuDjOnzwzzk9pBzKycoifGuUrHHcDixE/eLMKPHRaXenTgu1qXBqA==
+  dependencies:
+    commander "~1.1.1"
+
 jsdom@^21.1.0:
   version "21.1.0"
   resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-21.1.0.tgz#d56ba4a84ed478260d83bd53dc181775f2d8e6ef"
@@ -677,6 +770,11 @@ jsdom@^21.1.0:
     ws "^8.11.0"
     xml-name-validator "^4.0.0"
 
+keypress@0.1.x:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/keypress/-/keypress-0.1.0.tgz#4a3188d4291b66b4f65edb99f806aa9ae293592a"
+  integrity sha512-x0yf9PL/nx9Nw9oLL8ZVErFAk85/lslwEP7Vz7s5SI1ODXZIgit3C5qyWjw4DxOuO/3Hb4866SQh28a1V1d+WA==
+
 kuler@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3"
@@ -744,6 +842,13 @@ mime@1.6.0:
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
   integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
 
+minimatch@^3.0.4:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
+  integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
+  dependencies:
+    brace-expansion "^1.1.7"
+
 ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -771,6 +876,13 @@ node-fetch@2:
   dependencies:
     whatwg-url "^5.0.0"
 
+node-fetch@^2.6.12:
+  version "2.6.12"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.12.tgz#02eb8e22074018e3d5a83016649d04df0e348fba"
+  integrity sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 nwsapi@^2.2.2:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.2.tgz#e5418863e7905df67d51ec95938d67bf801f0bb0"
@@ -1009,7 +1121,7 @@ source-map@~0.6.1:
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
 
-stack-trace@0.0.x:
+stack-trace@0.0.x, stack-trace@^0.0.10:
   version "0.0.10"
   resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"
   integrity sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==
@@ -1179,7 +1291,7 @@ whatwg-url@^5.0.0:
     tr46 "~0.0.3"
     webidl-conversions "^3.0.0"
 
-winston-transport@^4.5.0:
+winston-transport@^4.3.0, winston-transport@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.5.0.tgz#6e7b0dd04d393171ed5e4e4905db265f7ab384fa"
   integrity sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==