Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
FFmpeg processor & updated .gitlab-ci.yml
All FFmpeg classes now inherit from "FFmpeg" abstract class, which makes it simpler to do not have duplicated code and bug tracking. In addition, when the YouTube data is downloaded, by using FFmpeg, the program removes the video, if exists, and normalizes the audio. If tests go well, we can mark audio milestone as completed.
  • Loading branch information
Javinator9889 committed Oct 9, 2019
1 parent 3e5bf8f commit dd6d174
Show file tree
Hide file tree
Showing 9 changed files with 90 additions and 41 deletions.
5 changes: 2 additions & 3 deletions .gitlab-ci.yml
Expand Up @@ -18,9 +18,8 @@ cache:

before_script:
- python -V # Print out python version for debugging
- apt update
- apt install -y libchromaprint-tools --install-recommends
- apt install -y ffmpeg --install-recommends
- apt update && apt upgrade -y
- apt install -y libchromaprint-tools ffmpeg --install-recommends
- pip install -r YouTubeMDBot/requirements.txt

test:pylint:
Expand Down
1 change: 1 addition & 0 deletions YouTubeMDBot/audio/__init__.py
Expand Up @@ -17,4 +17,5 @@
from ..audio.ffmpeg import FFmpegOGG
from ..audio.ffmpeg import FFmpegMP3
from ..audio.ffmpeg import ffmpeg_available
from ..audio.ffmpeg import FFmpegProcessor
from ..audio.fpcalc import FPCalc
77 changes: 45 additions & 32 deletions YouTubeMDBot/audio/ffmpeg.py
Expand Up @@ -13,10 +13,17 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from abc import ABC
from abc import abstractmethod
from io import BytesIO
from typing import List
from subprocess import PIPE
from subprocess import Popen

from ..constants import FFMPEG_OPENER
from ..constants import FFMPEG_CONVERTER
from ..constants import FFMPEG_PROCESSOR


def ffmpeg_available() -> bool:
try:
Expand All @@ -30,17 +37,23 @@ def ffmpeg_available() -> bool:
return proc.returncode == 0


class FFmpegOpener(object):
def __init__(self, data: bytes):
class FFmpeg(ABC):
def __init__(self, data: bytes, command: List[str] = None):
self._data = data
self.__ffmpeg_proc = Popen(["ffmpeg", "-i", "-", "-f", "s16le", "-"],
stdout=PIPE, stderr=PIPE, stdin=PIPE)
self.__command = command
self.__out = None
self.__err = None

def open(self) -> int:
self.__out, self.__err = self.__ffmpeg_proc.communicate(self._data)
return self.__ffmpeg_proc.returncode
def process(self) -> int:
proc = Popen(self.__command, stdout=PIPE, stderr=PIPE, stdin=PIPE)
self.__out, self.__err = proc.communicate(self._data)
return proc.returncode

def get_command(self) -> List[str]:
return self.__command

def set_command(self, command: List[str]):
self.__command = command

def get_output(self) -> bytes:
return self.__out
Expand All @@ -49,49 +62,49 @@ def get_extra(self) -> bytes:
return self.__err


class FFmpegExporter:
def __init__(self, data: BytesIO):
self._data = data
self.__command = ["ffmpeg", "-i", "-", "-vn", "-map_metadata", "0",
"-movflags", "use_metadata_tags"]
self.__out = None
self.__err = None
class FFmpegProcessor(FFmpeg):
def __init__(self, data: bytes):
super().__init__(data=data, command=FFMPEG_PROCESSOR.copy())

def _call_ffmpeg(self):
self._data.seek(0)
proc = Popen(self.__command, stdout=PIPE, stderr=PIPE, stdin=PIPE)
self.__out, self.__err = proc.communicate(self._data.read())

def _get_command(self) -> list:
return self.__command
class FFmpegOpener(FFmpeg):
def __init__(self, data: bytes):
super().__init__(data=data, command=FFMPEG_OPENER.copy())

def convert(self):
raise NotImplementedError

def get_output(self) -> bytes:
return self.__out
class FFmpegExporter(FFmpeg):
def __init__(self, data: bytes, bitrate: str = None):
super().__init__(data=data, command=FFMPEG_CONVERTER.copy())
self._bitrate = bitrate

def get_err(self) -> bytes:
return self.__err
@abstractmethod
def convert(self) -> int:
raise NotImplementedError


class FFmpegMP3(FFmpegExporter):
def convert(self):
command = super()._get_command()
def convert(self) -> int:
command = super().get_command()
if self._bitrate:
command.append("-b:a")
command.append(self._bitrate)
command.append("-acodec")
command.append("libmp3lame")
command.append("-f")
command.append("mp3")
command.append("-")
self._call_ffmpeg()
return self.process()


class FFmpegOGG(FFmpegExporter):
def convert(self):
command = super()._get_command()
def convert(self) -> int:
command = super().get_command()
if self._bitrate:
command.append("-b:a")
command.append(self._bitrate)
command.append("-c:a")
command.append("libvorbis")
command.append("-f")
command.append("ogg")
command.append("-")
self._call_ffmpeg()
return self.process()
3 changes: 3 additions & 0 deletions YouTubeMDBot/constants/__init__.py
Expand Up @@ -17,3 +17,6 @@
from ..constants.app_constants import FPCALC
from ..constants.app_constants import YDL_CLI_OPTIONS
from ..constants.app_constants import YOUTUBE
from ..constants.app_constants import FFMPEG_OPENER
from ..constants.app_constants import FFMPEG_PROCESSOR
from ..constants.app_constants import FFMPEG_CONVERTER
6 changes: 6 additions & 0 deletions YouTubeMDBot/constants/app_constants.py
Expand Up @@ -34,3 +34,9 @@
"https://www.googleapis.com/youtube/v3/videos?"
"part=id,snippet,contentDetails,statistics&id={0}&key={1}"
}

# FFmpeg commands
FFMPEG_OPENER = "ffmpeg -i - -f s16le -".split(" ")
FFMPEG_PROCESSOR = "ffmpeg -i - -filter:a loudnorm -vn -b:a 128k -f m4a -".split(" ")
FFMPEG_CONVERTER = ["ffmpeg", "-i", "-", "-vn", "-map_metadata", "0",
"-movflags", "use_metadata_tags"]
13 changes: 10 additions & 3 deletions YouTubeMDBot/downloader/youtube_downloader.py
Expand Up @@ -16,8 +16,9 @@
from io import BytesIO
from typing import Tuple

from ..errors import ProcessorError
from ..audio import FFmpegProcessor
from ..constants.app_constants import YDL_CLI_OPTIONS
from ..audio.ffmpeg import FFmpegOpener


class YouTubeDownloader(object):
Expand All @@ -26,7 +27,7 @@ def __init__(self, url: str):
self.__options: list = YDL_CLI_OPTIONS.copy()
self.__options.append(self.__url)

def download(self, ffmpeg: bool = False) -> Tuple[BytesIO, bytes]:
def download(self) -> Tuple[BytesIO, bytes]:
import subprocess

proc = subprocess.Popen(self.__options,
Expand All @@ -35,7 +36,13 @@ def download(self, ffmpeg: bool = False) -> Tuple[BytesIO, bytes]:
stdout, stderr = proc.communicate()
retcode = proc.returncode
if retcode == 0:
return BytesIO(stdout), stdout
processor = FFmpegProcessor(data=stdout)
if processor.process() == 0:
return BytesIO(processor.get_output()), processor.get_output()
else:
raise ProcessorError(
"ffmpeg failed: " + str(processor.get_extra().decode("utf-8"))
)
else:
raise RuntimeError("youtube-dl downloader exception - more info: " +
str(stderr.decode("utf-8")))
Expand Down
20 changes: 20 additions & 0 deletions YouTubeMDBot/errors/ProcessorError.py
@@ -0,0 +1,20 @@
# YouTubeMDBot
# Copyright (C) 2019 - Javinator9889
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.


class ProcessorError(Exception):
"""Raises an exception when FFmpeg processing fails"""
pass
1 change: 1 addition & 0 deletions YouTubeMDBot/errors/__init__.py
Expand Up @@ -15,3 +15,4 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from ..errors.EmptyBodyError import EmptyBodyError
from ..errors.NoMatchError import NoMatchError
from ..errors.ProcessorError import ProcessorError
5 changes: 2 additions & 3 deletions YouTubeMDBot/tests/converter.py
Expand Up @@ -14,11 +14,10 @@ class MyTestCase(TaggerTest):
def find_metadata(self, downloader: YouTubeDownloader) -> Tuple[BytesIO, bytes]:
io, data = super().find_metadata(downloader)
io.seek(0)
mp3 = FFmpegMP3(data=io)
ogg = FFmpegOGG(data=io)
mp3 = FFmpegMP3(data=data, bitrate="96k") # downrate
ogg = FFmpegOGG(data=data, bitrate="256k") # uprate

mp3.convert()
io.seek(0)
ogg.convert()

mp3_container = BytesIO(mp3.get_output())
Expand Down

0 comments on commit dd6d174

Please sign in to comment.