From dd6d17479468702e759b67596ce8d0339e18a2a4 Mon Sep 17 00:00:00 2001 From: Javinator9889 Date: Wed, 9 Oct 2019 11:21:13 +0200 Subject: [PATCH] FFmpeg processor & updated .gitlab-ci.yml All FFmpeg classes now inherit from "FFmpeg" abstract class, which makes it simpler to do not have duplicated code and bug tracking. In addition, when the YouTube data is downloaded, by using FFmpeg, the program removes the video, if exists, and normalizes the audio. If tests go well, we can mark audio milestone as completed. --- .gitlab-ci.yml | 5 +- YouTubeMDBot/audio/__init__.py | 1 + YouTubeMDBot/audio/ffmpeg.py | 77 +++++++++++-------- YouTubeMDBot/constants/__init__.py | 3 + YouTubeMDBot/constants/app_constants.py | 6 ++ YouTubeMDBot/downloader/youtube_downloader.py | 13 +++- YouTubeMDBot/errors/ProcessorError.py | 20 +++++ YouTubeMDBot/errors/__init__.py | 1 + YouTubeMDBot/tests/converter.py | 5 +- 9 files changed, 90 insertions(+), 41 deletions(-) create mode 100644 YouTubeMDBot/errors/ProcessorError.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2b6970..7005c2c 100755 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,9 +18,8 @@ cache: before_script: - python -V # Print out python version for debugging - - apt update - - apt install -y libchromaprint-tools --install-recommends - - apt install -y ffmpeg --install-recommends + - apt update && apt upgrade -y + - apt install -y libchromaprint-tools ffmpeg --install-recommends - pip install -r YouTubeMDBot/requirements.txt test:pylint: diff --git a/YouTubeMDBot/audio/__init__.py b/YouTubeMDBot/audio/__init__.py index 49eab31..7970b6b 100755 --- a/YouTubeMDBot/audio/__init__.py +++ b/YouTubeMDBot/audio/__init__.py @@ -17,4 +17,5 @@ from ..audio.ffmpeg import FFmpegOGG from ..audio.ffmpeg import FFmpegMP3 from ..audio.ffmpeg import ffmpeg_available +from ..audio.ffmpeg import FFmpegProcessor from ..audio.fpcalc import FPCalc diff --git a/YouTubeMDBot/audio/ffmpeg.py b/YouTubeMDBot/audio/ffmpeg.py index 0bf9930..bda56ef 100755 --- a/YouTubeMDBot/audio/ffmpeg.py +++ b/YouTubeMDBot/audio/ffmpeg.py @@ -13,10 +13,17 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from abc import ABC +from abc import abstractmethod from io import BytesIO +from typing import List from subprocess import PIPE from subprocess import Popen +from ..constants import FFMPEG_OPENER +from ..constants import FFMPEG_CONVERTER +from ..constants import FFMPEG_PROCESSOR + def ffmpeg_available() -> bool: try: @@ -30,17 +37,23 @@ def ffmpeg_available() -> bool: return proc.returncode == 0 -class FFmpegOpener(object): - def __init__(self, data: bytes): +class FFmpeg(ABC): + def __init__(self, data: bytes, command: List[str] = None): self._data = data - self.__ffmpeg_proc = Popen(["ffmpeg", "-i", "-", "-f", "s16le", "-"], - stdout=PIPE, stderr=PIPE, stdin=PIPE) + self.__command = command self.__out = None self.__err = None - def open(self) -> int: - self.__out, self.__err = self.__ffmpeg_proc.communicate(self._data) - return self.__ffmpeg_proc.returncode + def process(self) -> int: + proc = Popen(self.__command, stdout=PIPE, stderr=PIPE, stdin=PIPE) + self.__out, self.__err = proc.communicate(self._data) + return proc.returncode + + def get_command(self) -> List[str]: + return self.__command + + def set_command(self, command: List[str]): + self.__command = command def get_output(self) -> bytes: return self.__out @@ -49,49 +62,49 @@ def get_extra(self) -> bytes: return self.__err -class FFmpegExporter: - def __init__(self, data: BytesIO): - self._data = data - self.__command = ["ffmpeg", "-i", "-", "-vn", "-map_metadata", "0", - "-movflags", "use_metadata_tags"] - self.__out = None - self.__err = None +class FFmpegProcessor(FFmpeg): + def __init__(self, data: bytes): + super().__init__(data=data, command=FFMPEG_PROCESSOR.copy()) - def _call_ffmpeg(self): - self._data.seek(0) - proc = Popen(self.__command, stdout=PIPE, stderr=PIPE, stdin=PIPE) - self.__out, self.__err = proc.communicate(self._data.read()) - def _get_command(self) -> list: - return self.__command +class FFmpegOpener(FFmpeg): + def __init__(self, data: bytes): + super().__init__(data=data, command=FFMPEG_OPENER.copy()) - def convert(self): - raise NotImplementedError - def get_output(self) -> bytes: - return self.__out +class FFmpegExporter(FFmpeg): + def __init__(self, data: bytes, bitrate: str = None): + super().__init__(data=data, command=FFMPEG_CONVERTER.copy()) + self._bitrate = bitrate - def get_err(self) -> bytes: - return self.__err + @abstractmethod + def convert(self) -> int: + raise NotImplementedError class FFmpegMP3(FFmpegExporter): - def convert(self): - command = super()._get_command() + def convert(self) -> int: + command = super().get_command() + if self._bitrate: + command.append("-b:a") + command.append(self._bitrate) command.append("-acodec") command.append("libmp3lame") command.append("-f") command.append("mp3") command.append("-") - self._call_ffmpeg() + return self.process() class FFmpegOGG(FFmpegExporter): - def convert(self): - command = super()._get_command() + def convert(self) -> int: + command = super().get_command() + if self._bitrate: + command.append("-b:a") + command.append(self._bitrate) command.append("-c:a") command.append("libvorbis") command.append("-f") command.append("ogg") command.append("-") - self._call_ffmpeg() + return self.process() diff --git a/YouTubeMDBot/constants/__init__.py b/YouTubeMDBot/constants/__init__.py index 4b9bf4d..01e666d 100755 --- a/YouTubeMDBot/constants/__init__.py +++ b/YouTubeMDBot/constants/__init__.py @@ -17,3 +17,6 @@ from ..constants.app_constants import FPCALC from ..constants.app_constants import YDL_CLI_OPTIONS from ..constants.app_constants import YOUTUBE +from ..constants.app_constants import FFMPEG_OPENER +from ..constants.app_constants import FFMPEG_PROCESSOR +from ..constants.app_constants import FFMPEG_CONVERTER diff --git a/YouTubeMDBot/constants/app_constants.py b/YouTubeMDBot/constants/app_constants.py index e05c7a6..5f1a98d 100755 --- a/YouTubeMDBot/constants/app_constants.py +++ b/YouTubeMDBot/constants/app_constants.py @@ -34,3 +34,9 @@ "https://www.googleapis.com/youtube/v3/videos?" "part=id,snippet,contentDetails,statistics&id={0}&key={1}" } + +# FFmpeg commands +FFMPEG_OPENER = "ffmpeg -i - -f s16le -".split(" ") +FFMPEG_PROCESSOR = "ffmpeg -i - -filter:a loudnorm -vn -b:a 128k -f m4a -".split(" ") +FFMPEG_CONVERTER = ["ffmpeg", "-i", "-", "-vn", "-map_metadata", "0", + "-movflags", "use_metadata_tags"] diff --git a/YouTubeMDBot/downloader/youtube_downloader.py b/YouTubeMDBot/downloader/youtube_downloader.py index 1692b34..27552af 100755 --- a/YouTubeMDBot/downloader/youtube_downloader.py +++ b/YouTubeMDBot/downloader/youtube_downloader.py @@ -16,8 +16,9 @@ from io import BytesIO from typing import Tuple +from ..errors import ProcessorError +from ..audio import FFmpegProcessor from ..constants.app_constants import YDL_CLI_OPTIONS -from ..audio.ffmpeg import FFmpegOpener class YouTubeDownloader(object): @@ -26,7 +27,7 @@ def __init__(self, url: str): self.__options: list = YDL_CLI_OPTIONS.copy() self.__options.append(self.__url) - def download(self, ffmpeg: bool = False) -> Tuple[BytesIO, bytes]: + def download(self) -> Tuple[BytesIO, bytes]: import subprocess proc = subprocess.Popen(self.__options, @@ -35,7 +36,13 @@ def download(self, ffmpeg: bool = False) -> Tuple[BytesIO, bytes]: stdout, stderr = proc.communicate() retcode = proc.returncode if retcode == 0: - return BytesIO(stdout), stdout + processor = FFmpegProcessor(data=stdout) + if processor.process() == 0: + return BytesIO(processor.get_output()), processor.get_output() + else: + raise ProcessorError( + "ffmpeg failed: " + str(processor.get_extra().decode("utf-8")) + ) else: raise RuntimeError("youtube-dl downloader exception - more info: " + str(stderr.decode("utf-8"))) diff --git a/YouTubeMDBot/errors/ProcessorError.py b/YouTubeMDBot/errors/ProcessorError.py new file mode 100644 index 0000000..703bd49 --- /dev/null +++ b/YouTubeMDBot/errors/ProcessorError.py @@ -0,0 +1,20 @@ +# YouTubeMDBot +# Copyright (C) 2019 - Javinator9889 +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class ProcessorError(Exception): + """Raises an exception when FFmpeg processing fails""" + pass diff --git a/YouTubeMDBot/errors/__init__.py b/YouTubeMDBot/errors/__init__.py index 9fc7566..d9dd503 100755 --- a/YouTubeMDBot/errors/__init__.py +++ b/YouTubeMDBot/errors/__init__.py @@ -15,3 +15,4 @@ # along with this program. If not, see . from ..errors.EmptyBodyError import EmptyBodyError from ..errors.NoMatchError import NoMatchError +from ..errors.ProcessorError import ProcessorError diff --git a/YouTubeMDBot/tests/converter.py b/YouTubeMDBot/tests/converter.py index 9d7db7f..4901445 100644 --- a/YouTubeMDBot/tests/converter.py +++ b/YouTubeMDBot/tests/converter.py @@ -14,11 +14,10 @@ class MyTestCase(TaggerTest): def find_metadata(self, downloader: YouTubeDownloader) -> Tuple[BytesIO, bytes]: io, data = super().find_metadata(downloader) io.seek(0) - mp3 = FFmpegMP3(data=io) - ogg = FFmpegOGG(data=io) + mp3 = FFmpegMP3(data=data, bitrate="96k") # downrate + ogg = FFmpegOGG(data=data, bitrate="256k") # uprate mp3.convert() - io.seek(0) ogg.convert() mp3_container = BytesIO(mp3.get_output())