From fed2c7fda378fdf3c2821ed2e3de32b3400221ae Mon Sep 17 00:00:00 2001 From: Jesse Ashmore <29103230+JeeZeh@users.noreply.github.com> Date: Sun, 9 Aug 2020 15:56:14 +0100 Subject: [PATCH] Fixed some directory issues. Added UI download bar and API endpoint --- README.md | 4 +- package-lock.json | 15 +++++ package.json | 4 +- server.ts | 50 ++++++++++++--- src/components/App.tsx | 60 +++++++++++++----- src/components/DownloadBar.tsx | 94 +++++++++++++++++++++++++++++ src/components/PlaylistApi.ts | 9 ++- src/components/PlaylistExplorer.tsx | 46 +++++++------- src/components/Playlists.tsx | 7 +++ ytdl.ts | 18 ++++-- 10 files changed, 256 insertions(+), 51 deletions(-) create mode 100644 src/components/DownloadBar.tsx diff --git a/README.md b/README.md index 3121897..47d6c70 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ If you've ever wanted to quickly search for something that was said in a series # Caveats - these are important +> I **highly** recommend using a VPN to avoid being IP limited or banned. + 1. You _must_ provide a playlist. Even if you just want one video, wrap it in a playlist! 1. YouTube does a lot with playlists. For example, clicking "Play all" on a channel uploads will give you a playlist link. This is how you index an entire channel. 2. The subtitles are, in most cases, provided by YouTube's auto-captions, though it does download official ones if present! @@ -65,7 +67,7 @@ If you've ever wanted to quickly search for something that was said in a series 1. ~Figure out how to efficiently search for text across millions of subtitle entries~ [FlexSearch](https://github.com/nextapps-de/flexsearch) 1. ~Refresh playlist_data folder on changes so you don't need to keep restarting the UI~ 1. Include playlist title in Ytd -1. Download playlists from UI +1. ~Download playlists from UI~ ### UI diff --git a/package-lock.json b/package-lock.json index 9ff2149..c74367d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3635,6 +3635,11 @@ } } }, + "classnames": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.2.6.tgz", + "integrity": "sha512-JR/iSQOSt+LQIWwrwEzJ9uk0xfN3mTVYMwt1Ir5mUcSN6pU+V4zQFFaJsclJbPuAUQH+yfWef6tm7l1quW3C8Q==" + }, "clean-css": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.3.tgz", @@ -10884,6 +10889,16 @@ "@emotion/core": "^10.0.15" } }, + "react-toastify": { + "version": "6.0.8", + "resolved": "https://registry.npmjs.org/react-toastify/-/react-toastify-6.0.8.tgz", + "integrity": "sha512-NSqCNwv+C4IfR+c92PFZiNyeBwOJvigrP2bcRi2f6Hg3WqcHhEHOknbSQOs9QDFuqUjmK3SOrdvScQ3z63ifXg==", + "requires": { + "classnames": "^2.2.6", + "prop-types": "^15.7.2", + "react-transition-group": "^4.4.1" + } + }, "react-transition-group": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.1.tgz", diff --git a/package.json b/package.json index bf38463..df53958 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "react-feather": "^2.0.8", "react-router-dom": "^5.2.0", "react-spinners": "^0.9.0", + "react-toastify": "^6.0.8", "react-window": "^1.8.5", "styled-components": "^5.0.1", "typescript": "^3.9.7", @@ -70,7 +71,8 @@ "test": "echo \"Error: no test specified\" && exit 1", "tsc": "tsc -p .", "clean-dist": "rimraf dist/*", - "start": "npm run tsc && concurrently --kill-others \"webpack-dev-server --config=configs/webpack/dev.js\" \"node server.js\"", + "dev": "npm run tsc && concurrently --kill-others \"webpack-dev-server --config=configs/webpack/dev.js\" \"node server.js\"", + "start": "npm run tsc && npm run build && start http://localhost:6060 & node server.js", "server": "npm run tsc && node server.js", "build": "npm run clean-dist && webpack -p --config=configs/webpack/dev.js", "ytdl": "npm run update-ytdl && npm run tsc && node ytdl -- ", diff --git a/server.ts b/server.ts index f645a60..31009fd 100644 --- a/server.ts +++ b/server.ts @@ -1,17 +1,18 @@ +import { PlaylistJSON, YtdlPlaylistDownloader } from "./ytdl"; import express from "express"; -import { PlaylistJSON } from "./ytdl"; import * as fs from "fs"; import humanizeDuration from "humanize-duration"; import cors from "cors"; import md5 from "object-hash"; const app = express(); -const port = 8080; // default port to listen +const port = 6060; // default port to listen const playlistDirectory = "./playlist_data"; const playlists = new Map(); const playlistHashes = new Map(); const playlistMetadata = new Map(); +let ytdlSingleton = false; -app.use(cors()); +app.use(cors(), express.static("dist", {})); const generateMetadata = async (playlistData: PlaylistJSON) => { if (Object.keys(playlistData.videos).length === 0) return false; @@ -43,7 +44,7 @@ const initPlaylistData = async () => { fs.readFileSync(playlistDirectory + "/" + filename, "utf-8") ); metadataPromise.push(generateMetadata(playlistData)); - playlistHashes.set(filename, md5(playlistData.videos)); + playlistHashes.set(filename.split(".json")[0], md5(playlistData.videos)); playlists.set(filename.split(".json")[0], playlistData); } await Promise.all(metadataPromise); @@ -75,9 +76,10 @@ const checkUpdates = async (id?: string) => { await Promise.all(readAsync); for (const file of playlistDir) { let data = readAsync.find((d) => file === d.id + ".json"); - - if (md5(data.videos) !== playlistHashes.get(file)) { - playlists.set(file, data); + const filename = file.split(".json")[0]; + const hash = playlistHashes.get(filename); + if (!hash || md5(data.videos) !== hash) { + playlists.set(filename, data); toUpdate.push(data); } } @@ -129,6 +131,40 @@ const initRoutes = async () => { } res.json(playlist); }); + + app.post("/download", (req, res) => { + const id = req.query.id; + if (typeof id !== "string") { + res.status(400).send("Please provide a single ID string"); + return; + } + if (ytdlSingleton) + return res.status(400).json({ + success: false, + message: "Ytdl download already in progress", + }); + ytdlSingleton = true; + YtdlPlaylistDownloader(id) + .then((playlistId) => { + if (playlistId) { + res.json({ success: true, message: playlistId }); + } else { + res.status(500).json({ + success: false, + message: "Downloading failed, check server output", + }); + } + }) + .catch((e) => { + console.error(e); + res.status(500).json({ + success: false, + message: "Downloading failed, check server output", + error: e, + }); + }) + .finally(() => (ytdlSingleton = false)); + }); }; initPlaylistData() diff --git a/src/components/App.tsx b/src/components/App.tsx index 6b66c7d..26ec505 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -3,17 +3,39 @@ import { BrowserRouter as Router, Route, Link } from "react-router-dom"; import "./../assets/scss/App.scss"; import { createStyles, makeStyles, Theme } from "@material-ui/core/styles"; import { Playlists } from "./Playlists"; +import DownloadBar from "./DownloadBar"; import AppBar from "@material-ui/core/AppBar"; import Toolbar from "@material-ui/core/Toolbar"; import Typography from "@material-ui/core/Typography"; import PlaylistExplorer from "./PlaylistExplorer"; import styled from "styled-components"; import { Button } from "@material-ui/core"; +import { ToastContainer, toast } from "react-toastify"; +import "react-toastify/dist/ReactToastify.css"; +import { createMuiTheme } from "@material-ui/core/styles"; + +const theme = createMuiTheme({ + palette: { + primary: { + light: "#757ce8", + main: "#3f50b5", + dark: "#002884", + contrastText: "#fff", + }, + secondary: { + light: "#ff7961", + main: "#f44336", + dark: "#ba000d", + contrastText: "#000", + }, + }, +}); const useStyles = makeStyles((theme: Theme) => createStyles({ root: { flexGrow: 1, + backgroundColor: "#fafafa", minHeight: "101vh", }, @@ -23,6 +45,9 @@ const useStyles = makeStyles((theme: Theme) => header: { color: "whitesmoke", }, + bar: { + justifyContent: "space-between", + }, }) ); @@ -44,23 +69,30 @@ const App = (props: {}) => { return (
- - - - - YouTube Playlist Autosub Explorer + + +
+ + + YouTube Playlist Autosub Explorer + + +
+ + +
+ + - - - - +
+ diff --git a/src/components/DownloadBar.tsx b/src/components/DownloadBar.tsx new file mode 100644 index 0000000..95e402a --- /dev/null +++ b/src/components/DownloadBar.tsx @@ -0,0 +1,94 @@ +import * as React from "react"; +import { useState } from "react"; +import Typography from "@material-ui/core/Typography"; +import { SubSearchResult } from "./PlaylistExplorer"; +import { downloadPlaylist } from "./PlaylistApi"; +import { VideoInfo } from "../../ytdl"; +import { Play, Send } from "react-feather"; +import { + LinearProgress, + ListItemIcon, + TextField, + Button, + Grid, +} from "@material-ui/core"; +import ListItem from "@material-ui/core/ListItem"; +import ListItemText from "@material-ui/core/ListItemText"; +import { withStyles } from "@material-ui/core/styles"; +import PropTypes from "prop-types"; +import { IconButton } from "@material-ui/core/"; +import { Toast } from "react-toastify/dist/components"; + +const styles = { + root: { + maxWidth: 400, + justifyContent: "center", + alignItems: "center", + }, + input: { + width: 300, + color: "whitesmoke", + }, +}; + +const DownloadBar = (props: any) => { + const { classes, toastController } = props; + const [inputValue, setInputValue] = useState(""); + + const searchPlaylist = async () => { + if (inputValue === "") { + toastController.error("Please enter a playlist ID or URL", { + progress: 0, + autoClose: false, + }); + return; + } + + toastController.info( + "Attempting to download playlist. This might take a while, so check the console output for progress updates", + { progress: 0, autoClose: false } + ); + const download = await downloadPlaylist(inputValue); + if (!download.success) { + toastController.error(download.message, { + progress: 1, + autoClose: false, + }); + console.error("Error downloading playlist", download); + return; + } + + toastController.success("Download succeeded, refresh your playlists!", { + progress: 1, + autoClose: false, + }); + }; + + return ( + + + setInputValue(e.target.value)} + InputProps={{ + className: classes.input, + }} + /> + + + searchPlaylist()}> + + + + + ); +}; + +DownloadBar.propTypes = { + classes: PropTypes.object.isRequired, + toastController: PropTypes.func.isRequired, +}; + +export default withStyles(styles)(DownloadBar); diff --git a/src/components/PlaylistApi.ts b/src/components/PlaylistApi.ts index a0c0cf0..73e8dbc 100644 --- a/src/components/PlaylistApi.ts +++ b/src/components/PlaylistApi.ts @@ -1,9 +1,10 @@ import { PlaylistMetadata } from "../../server"; import { PlaylistJSON } from "../../ytdl"; -const host = "http://localhost:8080"; +const host = "http://localhost:6060"; const PLAYLIST_METADATA = "playlist_metadata"; const PLAYLIST = "playlist"; +const DOWNLOAD = "download"; export const getPlaylistMetadata = async (): Promise => { return await (await fetch(`${host}/${PLAYLIST_METADATA}`)).json(); @@ -12,3 +13,9 @@ export const getPlaylistMetadata = async (): Promise => { export const getPlaylist = async (id: string): Promise => { return await (await fetch(`${host}/${PLAYLIST}/${id}`)).json(); }; + +export const downloadPlaylist = async (id: string): Promise => { + return await ( + await fetch(`${host}/${DOWNLOAD}?id=${id}`, { method: "POST" }) + ).json(); +}; diff --git a/src/components/PlaylistExplorer.tsx b/src/components/PlaylistExplorer.tsx index 66da843..9f49a3e 100644 --- a/src/components/PlaylistExplorer.tsx +++ b/src/components/PlaylistExplorer.tsx @@ -105,8 +105,13 @@ const PlaylistExplorer = ({ * at 40, then we know subtitle 30 came from video 2 and it is the 10th subtitle. * @param playlist */ - const buildIndex = (): { index: Index; flatSubs: FlatSubs } => { + const buildIndex = (): { + index: Index; + flatSubs: FlatSubs; + } | null => { let flatSubs: FlatSubs = { idList: [], subs: [], idRange: [] }; + if (!playlist) return null; + console.log("Building index"); const index: Index = FlexSearch.create({ @@ -132,24 +137,17 @@ const PlaylistExplorer = ({ getPlaylist(playlistId).then((p) => { setPlaylist(p); }); - - return () => { - setPlaylist(null); - }; }, []); useEffect(() => { if (playlist && !flexIndex) { - const { index, flatSubs } = buildIndex(); + const builtIndex = buildIndex(); + if (!builtIndex) return; + const { index, flatSubs } = builtIndex; setFlexIndex(index); setFlatSubs(flatSubs); index.search(" ").then(() => setLoaded(true)); } - - return () => { - setFlatSubs(null); - setFlexIndex(null); - }; }, [playlist]); useEffect(() => { @@ -201,9 +199,9 @@ const PlaylistExplorer = ({ const renderRow = (props: ListChildComponentProps) => { const { index, style } = props; + if (!playlist || !flexResults) return <>; const video = playlist.videos[flexResults[index].videoId]; - if (!video) { - console.log("BROKE", flexResults[index].videoId); + if (!video || !style) { return <>; } return ( @@ -218,16 +216,18 @@ const PlaylistExplorer = ({ const FlexResults = () => { return ( - - - {renderRow} - - + flexResults && ( + + + {renderRow} + + + ) ); }; diff --git a/src/components/Playlists.tsx b/src/components/Playlists.tsx index 0ce6f0e..2300c93 100644 --- a/src/components/Playlists.tsx +++ b/src/components/Playlists.tsx @@ -109,6 +109,13 @@ export const Playlists = () => { {renderPlaylist(p)} ))} + {playlists && playlists.length === 0 && ( + + No playlists downloaded.
+ Use npm run ytdl [playlist] from the directory, or + paste a playlist link or ID the download bar above! + + )} {!playlists && } diff --git a/ytdl.ts b/ytdl.ts index b69b7e4..2829ac4 100644 --- a/ytdl.ts +++ b/ytdl.ts @@ -420,21 +420,29 @@ export const YtdlPlaylistDownloader = async (playlistId: string) => { JSON.stringify(playlist) ); console.log("Done!"); + return playlistId; } catch (error) { console.error("Error writing playlist data to file\n", error); + return; } } else { logTitle("Done! No changes made to playlist data."); } + return playlistId; }; const main = async () => { logTitle("YouTube Playlist Autosub Downloader"); const processedArgs: string[] = []; - const args = process.argv.slice( - process.argv.findIndex((a) => a === "--") + 1 - ); + const sliceIndex = process.argv.findIndex((a) => a === "--"); + + if (sliceIndex === -1) { + console.error("Program not launched using npm run ytdl!"); + return; + } + + const args = process.argv.slice(sliceIndex + 1); const threads = args.find((a) => a.includes("threads")); if (threads) { @@ -474,4 +482,6 @@ const main = async () => { } }; -main(); +if (require.main === module) { + main(); +}