Merge pull request #35 from kenu/transcript

🐛 transcript bug fix

.github/workflows/main.yml

@@ -15,4 +15,4 @@ jobs:
           key: ${{ secrets.KEY }}
           port: ${{ secrets.PORT }}
           script: |
-            ./scripts/deploy-youtb.sh
+            ./scripts/deploy-odevtube.sh

README.md

@@ -6,7 +6,7 @@
 - YouTube Data API v3를 활성화
 - 프로젝트에서 신규 API 키를 생성
   - 환경변수 YOUTUBE_API_KEY 설정
-- [package.json](https://github.com/kenu/youtb/blob/main/package.json)에 youtube-api를 추가 `"googleapis": "^134.0.0"`
+- [package.json](https://github.com/kenu/odevtube/blob/main/package.json)에 youtube-api를 추가 `"googleapis": "^134.0.0"`
 
 ```
   {

package.json

@@ -12,7 +12,8 @@
     "node-html-parser": "^6.1.13",
     "nodemailer": "^6.9.13",
     "openai": "^4.38.2",
-    "sequelize": "^6.37.3"
+    "sequelize": "^6.37.3",
+    "youtube-transcript": "^1.2.1"
   },
   "devDependencies": {
     "@babel/core": "^7.24.4",

scripts/deploy-scripts.sh

@@ -1,12 +1,12 @@
 #!/bin/zsh
 . ~/.zshrc
 
-cd ~/git/youtb
+cd ~/git/odevtube
 git pull origin main
 pnpm i
-cd ~/git/youtb/web
+cd ~/git/odevtube/web
 pnpm i
-pm2 restart youtb --update-env
+pm2 restart odevtube --update-env
 sleep 2
 pm2 list
 

tests/transcript.test.js

@@ -1,7 +1,21 @@
-import fetchTranscript from '../web/utils/transcript.js'
+import { YoutubeTranscript } from "youtube-transcript";
 
-test('', async() => {
-  const videoId = 'ulqHnefBFMM'
-  const transcript = await fetchTranscript(videoId)
-  expect(transcript).not.toBeNull()
-})
+const videoId = "vSIb2sPdu9U";
+test("getTextOnly", async () => {
+  const transcript = await YoutubeTranscript.fetchTranscript(videoId);
+  expect(transcript).not.toBeNull();
+  const fullText = getTextOnly(transcript);
+  expect(fullText).toContain("인데");
+});
+
+function getTextOnly(transcript) {
+  return transcript.map((item) => item.text).join(" ");
+}
+
+test('remove [Object, Object]', async () => {
+  const pattern = /(니다|하죠|하겠죠|고요|까요|네요|데요|세요|아요|어요)\s/g
+  const transcript = await YoutubeTranscript.fetchTranscript(videoId)
+  let fullText = transcript.map((item) => item.text).join(' ')
+  fullText = fullText.replaceAll(pattern, '$1. ')
+  expect(fullText).not.toContain('Object')
+});

web/package.json

@@ -4,7 +4,7 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "dev": "nodemon ./bin/youtb.js"
+    "dev": "nodemon ./bin/odevtube.js"
   },
   "dependencies": {
     "body-parser": "^1.20.2",

web/routes/index.js

@@ -1,6 +1,7 @@
 import express from 'express'
 import dayjs from 'dayjs'
 import passport from 'passport'
+import { YoutubeTranscript } from 'youtube-transcript'
 import dao from '../../youtubeDao.js'
 
 const router = express.Router()
@@ -119,7 +120,6 @@ function building(list) {
   })
 }
 
-import fetchTranscript from '../utils/transcript.js'
 import summarize from '../utils/summary.js'
 router.get('/transcript/:videoId', async function (req, res, next) {
   const videoId = req.params.videoId
@@ -129,33 +129,35 @@ router.get('/transcript/:videoId', async function (req, res, next) {
     res.json({ videoId, summary: item.summary, text: item.content })
     return
   }
-  // if empty get from youtube web
-  // save with videoId
+  await upsertTranscript(res, videoId)
+
+})
+
+async function upsertTranscript(res, videoId) {
   try {
-    const pattern = /(니다|하죠|하겠죠|고요|까요|네요|데요|세요|아요|어요)\s/g
-    let transcript = await fetchTranscript(videoId)
-    transcript = transcript.replaceAll(pattern, '$1. ')
-    const cmd =
-      "3줄 단문에, 명사형 어미로 요약(예)'있습니다.' 대신 '있음', '설명드립니다' 대신 '설명함' :\n"
+    const pattern = /(니다|이죠|하죠|하겠죠|고요|까요|네요|데요|세요|에요|아요|어요)\s/g
+    const transcript = await YoutubeTranscript.fetchTranscript(videoId)
+    let fullText = transcript.map((item) => item.text).join(' ')
+    fullText = fullText.replaceAll(pattern, '$1. ')
+    const cmd = "3줄 단문에, 명사형 어미로 요약(예)'있습니다.' 대신 '있음', '설명드립니다' 대신 '설명함' :\n"
     const messages = [
       {
         role: 'system',
-        content: cmd + transcript,
+        content: cmd + fullText,
       },
     ]
-    const result = await summarize(messages)
-    let summary = result[0].message.content
+    const summary = await summarize(messages)
     await dao.createTranscript({
       videoId,
-      content: transcript,
+      content: fullText,
       summary: summary,
     })
-    res.json({ videoId, summary, text: transcript })
+    res.json({ videoId, summary, text: fullText })
   } catch (error) {
     console.log(error)
     res.json({ videoId, summary: '', text: 'Not Available' })
   }
-})
+}
 
 router.get('/login', function (req, res) {
   res.render('login')

web/utils/summary.js

@@ -4,10 +4,10 @@ async function summarize(messages) {
   const openai = new OpenAI()
   const completion = await openai.chat.completions.create({
     messages: messages,
-    model: 'gpt-3.5-turbo',
+    model: 'gpt-4o-mini',
   })
 
-  return completion.choices
+  return completion.choices[0].message.content
 }
 
 export default summarize

web/views/admin/channel.ejs

@@ -23,7 +23,7 @@
       | <a href="/admin?c=food" alt="food">food</a> |</span>
     <a href="/admin" alt="korean">dev ko</a> |
     <a href="/admin?c=dev&l=en" alt="english">dev en</a>
-    <a href="https://github.com/kenu/youtb" target="_blank">
+    <a href="https://github.com/kenu/odevtube" target="_blank">
       <img src="/images/github-mark.svg" class="github-logo" alt="fork in github"></a>
   </aside>
 

web/views/admin/video.ejs

@@ -23,7 +23,7 @@
       | <a href="/admin?c=food" title="">food</a> |</span>
     <a href="/admin" title="korean">dev ko</a> |
     <a href="/admin?c=dev&l=en" title="english">dev en</a>
-    <a href="https://github.com/kenu/youtb" target="_blank">
+    <a href="https://github.com/kenu/odevtube" target="_blank">
       <img src="/images/github-mark.svg" class="github-logo" alt="fork in github"></a>
   </aside>
 

web/views/index.ejs

@@ -12,7 +12,7 @@
       </a>
     </h1>
     <div class="github">
-      <a href="https://github.com/kenu/youtb" target="_blank">
+      <a href="https://github.com/kenu/odevtube" target="_blank">
         <img src="/images/github-mark.svg" class="github-logo" alt="fork in github"></a>
     </div>
   </section>