preservetube
/
backend
mirror of https://github.com/preservetube/backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

logtail, catching errors while downloading playlists and channels, stop when websocket is closed

This commit is contained in:
localhost 2023-07-07 21:31:25 +02:00
parent 7159811489
commit 81830f0e2c
5 changed files with 159 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
controller/websocket.js
View File

@ -117,6 +117,10 @@ exports.playlist = async (ws, req) => {
async function startDownloading() {
const playlist = await metadata.getPlaylistVideos(playlistId)
for (video of playlist.relatedStreams) {
if (ws.readyState !== ws.OPEN) {
return logger.info({ message: `Stopped downloading ${playlistId}, websocket is closed` })
}
const id = video.url.match(/[?&]v=([^&]+)/)[1]
const already = await prisma.videos.findFirst({
@ -159,16 +163,21 @@ exports.playlist = async (ws, req) => {
} else {
const file = fs.readdirSync("./videos").find(f => f.includes(id))
if (file) {
fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
ws.send(`DATA - Downloaded ${video.title}`)
ws.send(`DATA - Uploading ${video.title}`)
try {
fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
ws.send(`DATA - Downloaded ${video.title}`)
ws.send(`DATA - Uploading ${video.title}`)
const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
ws.send(`DATA - Uploaded ${video.title}`)
fs.unlinkSync(`./videos/${id}.webm`)
const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
ws.send(`DATA - Uploaded ${video.title}`)
fs.unlinkSync(`./videos/${id}.webm`)
await websocket.createDatabaseVideo(id, videoUrl, playlistId)
ws.send(`DATA - Created video page for ${video.title}`)
await websocket.createDatabaseVideo(id, videoUrl, playlistId)
ws.send(`DATA - Created video page for ${video.title}`)
} catch (e) {
ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`)
logger.error(e)
}
} else {
ws.send(`DATA - Failed to find file for ${video.title}. Going to next video in the playlist`)
}
@ -212,6 +221,10 @@ exports.channel = async (ws, req) => {
const videos = await metadata.getChannelVideos(channelId)
for (video of videos) {
if (ws.readyState !== ws.OPEN) {
return logger.info({ message: `Stopped downloading ${channelId}, websocket is closed` })
}
const id = video.url.match(/[?&]v=([^&]+)/)[1]
const already = await prisma.videos.findFirst({
@ -246,16 +259,21 @@ exports.channel = async (ws, req) => {
} else {
const file = fs.readdirSync("./videos").find(f => f.includes(id))
if (file) {
fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
ws.send(`DATA - Downloaded ${video.title}`)
ws.send(`DATA - Uploading ${video.title}`)
try {
fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
ws.send(`DATA - Downloaded ${video.title}`)
ws.send(`DATA - Uploading ${video.title}`)
const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
ws.send(`DATA - Uploaded ${video.title}`)
fs.unlinkSync(`./videos/${id}.webm`)
const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
ws.send(`DATA - Uploaded ${video.title}`)
fs.unlinkSync(`./videos/${id}.webm`)
await websocket.createDatabaseVideo(id, videoUrl)
ws.send(`DATA - Created video page for ${video.title}`)
await websocket.createDatabaseVideo(id, videoUrl)
ws.send(`DATA - Created video page for ${video.title}`)
} catch (e) {
ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`)
logger.error(e)
}
} else {
ws.send(`DATA - Failed to find file for ${video.title}. Going to next video`)
}

2
index.js
View File

@ -34,7 +34,7 @@ app.ws('/savechannel', websocketController.channel)
app.get('/autodownload', websocketController.addAutodownload)
process.on('uncaughtException', err => {
logger.info({ message: `Error: ${err.message}` })
logger.error(err)
})
app.listen(1337, () => {

2
package.json
View File

@ -4,6 +4,8 @@
"main": "index.js",
"license": "AGPL-3.0",
"dependencies": {
"@logtail/node": "^0.4.0",
"@logtail/winston": "^0.4.1",
"@prisma/client": "4.9.0",
"aws-sdk": "2.1128.0",
"child_process": "^1.0.2",

8
utils/logger.js
View File

@ -1,4 +1,9 @@
const winston = require('winston')
const { Logtail } = require("@logtail/node")
const { LogtailTransport } = require("@logtail/winston")
const logtail = new Logtail("YFQdKmZgGvxPpusqxCSxsj2b")
const logger = winston.createLogger({
format: winston.format.json(),
transports: [
@ -13,6 +18,9 @@ const logger = winston.createLogger({
winston.format.timestamp({format: 'MMM-DD-YYYY HH:mm:ss'}),
winston.format.printf(info => `${[info.timestamp]}: ${info.message}`),
)}),
new LogtailTransport(logtail, {
level: 'error'
})
],
});

116
yarn.lock
View File

@ -21,6 +21,55 @@
resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==
"@logtail/core@^0.4.0":
version "0.4.0"
resolved "https://registry.yarnpkg.com/@logtail/core/-/core-0.4.0.tgz#a8e510756bf9fd1b2cc6c901225eb03685919ca8"
integrity sha512-xItEU5vY2LvcJyAVkxKKZ3UvG3CsK2pMvhuLDntlC+US1LlNcfQmVzvInN5j5iXl0kjNO6n+0EY2YyhhOgjWCQ==
dependencies:
"@logtail/tools" "^0.4.0"
"@logtail/types" "^0.4.0"
"@logtail/node@^0.4.0":
version "0.4.0"
resolved "https://registry.yarnpkg.com/@logtail/node/-/node-0.4.0.tgz#71488911a6dc4bf218871af80e3788cd727f8474"
integrity sha512-0/1to4HLf95FrDlZoFLz3A0Fz/B55G82syY22Yl9N4mjC2wW6t5Lo9d4q/u/VmHBHW3PBbytbmFEoHx0H5Sarg==
dependencies:
"@logtail/core" "^0.4.0"
"@logtail/types" "^0.4.0"
"@msgpack/msgpack" "^2.5.1"
"@types/stack-trace" "^0.0.29"
cross-fetch "^3.0.4"
minimatch "^3.0.4"
stack-trace "^0.0.10"
"@logtail/tools@^0.4.0":
version "0.4.0"
resolved "https://registry.yarnpkg.com/@logtail/tools/-/tools-0.4.0.tgz#ff1bed193a005b12007df510f6ce294b8f518207"
integrity sha512-7lSKdJTq7NMUvriMhkf6cTxY9QlXi+YLLuHxbSOKI56HpJN8qJGPVcmOdzlbk8pJofe9RnME8hBMLTHAD3bmsw==
dependencies:
"@logtail/types" "^0.4.0"
"@logtail/types@^0.4.0":
version "0.4.0"
resolved "https://registry.yarnpkg.com/@logtail/types/-/types-0.4.0.tgz#39c28e08f805657c70107071dc166866153563a8"
integrity sha512-2CR3w7Xf5rKWbUlgiaBKeVrqWuWnVBz0Ymycw/gYwupVI0fb6Ameid9fHfeM5LI/gbIO3ZTIMADlA0FvFbuXMQ==
dependencies:
js "^0.1.0"
"@logtail/winston@^0.4.1":
version "0.4.1"
resolved "https://registry.yarnpkg.com/@logtail/winston/-/winston-0.4.1.tgz#bbdb60db0ae1e013a1e9678ba25160eb2554fecf"
integrity sha512-YsMXzhcpGHDd3ijyqnDD+2qraJAiar/mVjR7HP8NYQsQMk9dQTIHbQzXkBUwAfVOsKSzNbfzeWTnft8sQBYxYQ==
dependencies:
"@logtail/node" "^0.4.0"
"@logtail/types" "^0.4.0"
winston-transport "^4.3.0"
"@msgpack/msgpack@^2.5.1":
version "2.8.0"
resolved "https://registry.yarnpkg.com/@msgpack/msgpack/-/msgpack-2.8.0.tgz#4210deb771ee3912964f14a15ddfb5ff877e70b9"
integrity sha512-h9u4u/jiIRKbq25PM+zymTyW6bhTzELvOoUd+AvYriWOAKpLGnIamaET3pnHYoI5iYphAHBI4ayx0MehR+VVPQ==
"@prisma/client@4.9.0":
version "4.9.0"
resolved "https://registry.yarnpkg.com/@prisma/client/-/client-4.9.0.tgz#4a4068f3540732ea5723c008d49ed684d20f9340"
@ -50,6 +99,11 @@
dependencies:
"@types/trusted-types" "*"
"@types/stack-trace@^0.0.29":
version "0.0.29"
resolved "https://registry.yarnpkg.com/@types/stack-trace/-/stack-trace-0.0.29.tgz#eb7a7c60098edb35630ed900742a5ecb20cfcb4d"
integrity sha512-TgfOX+mGY/NyNxJLIbDWrO9DjGoVSW9+aB8H2yy1fy32jsvxijhmyJI9fDFgvz3YP4lvJaq9DzdR/M1bOgVc9g==
"@types/triple-beam@^1.3.2":
version "1.3.2"
resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.2.tgz#38ecb64f01aa0d02b7c8f4222d7c38af6316fef8"
@ -128,6 +182,11 @@ aws-sdk@2.1128.0:
uuid "3.3.2"
xml2js "0.4.19"
balanced-match@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
base64-js@^1.0.2:
version "1.5.1"
resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
@ -151,6 +210,14 @@ body-parser@1.20.1:
type-is "~1.6.18"
unpipe "1.0.0"
brace-expansion@^1.1.7:
version "1.1.11"
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==
dependencies:
balanced-match "^1.0.0"
concat-map "0.0.1"
buffer@4.9.2:
version "4.9.2"
resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8"
@ -231,6 +298,18 @@ combined-stream@^1.0.8:
dependencies:
delayed-stream "~1.0.0"
commander@~1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/commander/-/commander-1.1.1.tgz#50d1651868ae60eccff0a2d9f34595376bc6b041"
integrity sha512-71Rod2AhcH3JhkBikVpNd0pA+fWsmAaVoti6OR38T76chA7vE3pSerS0Jor4wDw+tOueD2zLVvFOw5H0Rcj7rA==
dependencies:
keypress "0.1.x"
concat-map@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==
content-disposition@0.5.4:
version "0.5.4"
resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe"
@ -261,6 +340,13 @@ cors@^2.8.5:
object-assign "^4"
vary "^1"
cross-fetch@^3.0.4:
version "3.1.8"
resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82"
integrity sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==
dependencies:
node-fetch "^2.6.12"
cssom@^0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.5.0.tgz#d254fa92cd8b6fbd83811b9fbaed34663cc17c36"
@ -645,6 +731,13 @@ jmespath@0.16.0:
resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076"
integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==
js@^0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/js/-/js-0.1.0.tgz#e1d0afd55ea39c2b28da304e8143eaf2c133f366"
integrity sha512-ZBbGYOpact8QAH9RprFWL4RAESYwbDodxiuDjOnzwzzk9pBzKycoifGuUrHHcDixE/eLMKPHRaXenTgu1qXBqA==
dependencies:
commander "~1.1.1"
jsdom@^21.1.0:
version "21.1.0"
resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-21.1.0.tgz#d56ba4a84ed478260d83bd53dc181775f2d8e6ef"
@ -677,6 +770,11 @@ jsdom@^21.1.0:
ws "^8.11.0"
xml-name-validator "^4.0.0"
keypress@0.1.x:
version "0.1.0"
resolved "https://registry.yarnpkg.com/keypress/-/keypress-0.1.0.tgz#4a3188d4291b66b4f65edb99f806aa9ae293592a"
integrity sha512-x0yf9PL/nx9Nw9oLL8ZVErFAk85/lslwEP7Vz7s5SI1ODXZIgit3C5qyWjw4DxOuO/3Hb4866SQh28a1V1d+WA==
kuler@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3"
@ -744,6 +842,13 @@ mime@1.6.0:
resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
minimatch@^3.0.4:
version "3.1.2"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
dependencies:
brace-expansion "^1.1.7"
ms@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@ -771,6 +876,13 @@ node-fetch@2:
dependencies:
whatwg-url "^5.0.0"
node-fetch@^2.6.12:
version "2.6.12"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.12.tgz#02eb8e22074018e3d5a83016649d04df0e348fba"
integrity sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==
dependencies:
whatwg-url "^5.0.0"
nwsapi@^2.2.2:
version "2.2.2"
resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.2.tgz#e5418863e7905df67d51ec95938d67bf801f0bb0"
@ -1009,7 +1121,7 @@ source-map@~0.6.1:
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
stack-trace@0.0.x:
stack-trace@0.0.x, stack-trace@^0.0.10:
version "0.0.10"
resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"
integrity sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==
@ -1179,7 +1291,7 @@ whatwg-url@^5.0.0:
tr46 "~0.0.3"
webidl-conversions "^3.0.0"
winston-transport@^4.5.0:
winston-transport@^4.3.0, winston-transport@^4.5.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.5.0.tgz#6e7b0dd04d393171ed5e4e4905db265f7ab384fa"
integrity sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==