mirror of
https://github.com/clinton-hall/nzbToMedia.git
synced 2024-11-14 17:40:24 -08:00
f05b09f349
Updates rarfile to 3.1 Updates stevedore to 3.5.0 Updates appdirs to 1.4.4 Updates click to 8.1.3 Updates decorator to 5.1.1 Updates dogpile.cache to 1.1.8 Updates pbr to 5.11.0 Updates pysrt to 1.1.2 Updates pytz to 2022.6 Adds importlib-metadata version 3.1.1 Adds typing-extensions version 4.1.1 Adds zipp version 3.11.0
499 lines
21 KiB
Python
499 lines
21 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
Subliminal uses `click <http://click.pocoo.org>`_ to provide a powerful :abbr:`CLI (command-line interface)`.
|
|
|
|
"""
|
|
from __future__ import division
|
|
from collections import defaultdict
|
|
from datetime import timedelta
|
|
import glob
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
|
|
from appdirs import AppDirs
|
|
from babelfish import Error as BabelfishError, Language
|
|
import click
|
|
from dogpile.cache.backends.file import AbstractFileLock
|
|
from dogpile.util.readwrite_lock import ReadWriteMutex
|
|
from six.moves import configparser
|
|
|
|
from subliminal import (AsyncProviderPool, Episode, Movie, Video, __version__, check_video, compute_score, get_scores,
|
|
provider_manager, refine, refiner_manager, region, save_subtitles, scan_video, scan_videos)
|
|
from subliminal.core import ARCHIVE_EXTENSIONS, search_external_subtitles
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class MutexLock(AbstractFileLock):
|
|
""":class:`MutexLock` is a thread-based rw lock based on :class:`dogpile.core.ReadWriteMutex`."""
|
|
def __init__(self, filename):
|
|
self.mutex = ReadWriteMutex()
|
|
|
|
def acquire_read_lock(self, wait):
|
|
ret = self.mutex.acquire_read_lock(wait)
|
|
return wait or ret
|
|
|
|
def acquire_write_lock(self, wait):
|
|
ret = self.mutex.acquire_write_lock(wait)
|
|
return wait or ret
|
|
|
|
def release_read_lock(self):
|
|
return self.mutex.release_read_lock()
|
|
|
|
def release_write_lock(self):
|
|
return self.mutex.release_write_lock()
|
|
|
|
|
|
class Config(object):
|
|
"""A :class:`~configparser.ConfigParser` wrapper to store configuration.
|
|
|
|
Interaction with the configuration is done with the properties.
|
|
|
|
:param str path: path to the configuration file.
|
|
|
|
"""
|
|
def __init__(self, path):
|
|
#: Path to the configuration file
|
|
self.path = path
|
|
|
|
#: The underlying configuration object
|
|
self.config = configparser.SafeConfigParser()
|
|
self.config.add_section('general')
|
|
self.config.set('general', 'languages', json.dumps(['en']))
|
|
self.config.set('general', 'providers', json.dumps(sorted([p.name for p in provider_manager])))
|
|
self.config.set('general', 'refiners', json.dumps(sorted([r.name for r in refiner_manager])))
|
|
self.config.set('general', 'single', str(0))
|
|
self.config.set('general', 'embedded_subtitles', str(1))
|
|
self.config.set('general', 'age', str(int(timedelta(weeks=2).total_seconds())))
|
|
self.config.set('general', 'hearing_impaired', str(1))
|
|
self.config.set('general', 'min_score', str(0))
|
|
|
|
def read(self):
|
|
"""Read the configuration from :attr:`path`"""
|
|
self.config.read(self.path)
|
|
|
|
def write(self):
|
|
"""Write the configuration to :attr:`path`"""
|
|
with open(self.path, 'w') as f:
|
|
self.config.write(f)
|
|
|
|
@property
|
|
def languages(self):
|
|
return {Language.fromietf(l) for l in json.loads(self.config.get('general', 'languages'))}
|
|
|
|
@languages.setter
|
|
def languages(self, value):
|
|
self.config.set('general', 'languages', json.dumps(sorted([str(l) for l in value])))
|
|
|
|
@property
|
|
def providers(self):
|
|
return json.loads(self.config.get('general', 'providers'))
|
|
|
|
@providers.setter
|
|
def providers(self, value):
|
|
self.config.set('general', 'providers', json.dumps(sorted([p.lower() for p in value])))
|
|
|
|
@property
|
|
def refiners(self):
|
|
return json.loads(self.config.get('general', 'refiners'))
|
|
|
|
@refiners.setter
|
|
def refiners(self, value):
|
|
self.config.set('general', 'refiners', json.dumps([r.lower() for r in value]))
|
|
|
|
@property
|
|
def single(self):
|
|
return self.config.getboolean('general', 'single')
|
|
|
|
@single.setter
|
|
def single(self, value):
|
|
self.config.set('general', 'single', str(int(value)))
|
|
|
|
@property
|
|
def embedded_subtitles(self):
|
|
return self.config.getboolean('general', 'embedded_subtitles')
|
|
|
|
@embedded_subtitles.setter
|
|
def embedded_subtitles(self, value):
|
|
self.config.set('general', 'embedded_subtitles', str(int(value)))
|
|
|
|
@property
|
|
def age(self):
|
|
return timedelta(seconds=self.config.getint('general', 'age'))
|
|
|
|
@age.setter
|
|
def age(self, value):
|
|
self.config.set('general', 'age', str(int(value.total_seconds())))
|
|
|
|
@property
|
|
def hearing_impaired(self):
|
|
return self.config.getboolean('general', 'hearing_impaired')
|
|
|
|
@hearing_impaired.setter
|
|
def hearing_impaired(self, value):
|
|
self.config.set('general', 'hearing_impaired', str(int(value)))
|
|
|
|
@property
|
|
def min_score(self):
|
|
return self.config.getfloat('general', 'min_score')
|
|
|
|
@min_score.setter
|
|
def min_score(self, value):
|
|
self.config.set('general', 'min_score', str(value))
|
|
|
|
@property
|
|
def provider_configs(self):
|
|
rv = {}
|
|
for provider in provider_manager:
|
|
if self.config.has_section(provider.name):
|
|
rv[provider.name] = {k: v for k, v in self.config.items(provider.name)}
|
|
return rv
|
|
|
|
@provider_configs.setter
|
|
def provider_configs(self, value):
|
|
# loop over provider configurations
|
|
for provider, config in value.items():
|
|
# create the corresponding section if necessary
|
|
if not self.config.has_section(provider):
|
|
self.config.add_section(provider)
|
|
|
|
# add config options
|
|
for k, v in config.items():
|
|
self.config.set(provider, k, v)
|
|
|
|
@property
|
|
def refiner_configs(self):
|
|
rv = {}
|
|
for refiner in refiner_manager:
|
|
if self.config.has_section(refiner.name):
|
|
rv[refiner.name] = {k: v for k, v in self.config.items(refiner.name)}
|
|
return rv
|
|
|
|
@refiner_configs.setter
|
|
def refiner_configs(self, value):
|
|
# loop over refiner configurations
|
|
for refiner, config in value.items():
|
|
# create the corresponding section if necessary
|
|
if not self.config.has_section(refiner):
|
|
self.config.add_section(refiner)
|
|
|
|
# add config options
|
|
for k, v in config.items():
|
|
self.config.set(refiner, k, v)
|
|
|
|
|
|
class LanguageParamType(click.ParamType):
|
|
""":class:`~click.ParamType` for languages that returns a :class:`~babelfish.language.Language`"""
|
|
name = 'language'
|
|
|
|
def convert(self, value, param, ctx):
|
|
try:
|
|
return Language.fromietf(value)
|
|
except BabelfishError:
|
|
self.fail('%s is not a valid language' % value)
|
|
|
|
|
|
LANGUAGE = LanguageParamType()
|
|
|
|
|
|
class AgeParamType(click.ParamType):
|
|
""":class:`~click.ParamType` for age strings that returns a :class:`~datetime.timedelta`
|
|
|
|
An age string is in the form `number + identifier` with possible identifiers:
|
|
|
|
* ``w`` for weeks
|
|
* ``d`` for days
|
|
* ``h`` for hours
|
|
|
|
The form can be specified multiple times but only with that idenfier ordering. For example:
|
|
|
|
* ``1w2d4h`` for 1 week, 2 days and 4 hours
|
|
* ``2w`` for 2 weeks
|
|
* ``3w6h`` for 3 weeks and 6 hours
|
|
|
|
"""
|
|
name = 'age'
|
|
|
|
def convert(self, value, param, ctx):
|
|
match = re.match(r'^(?:(?P<weeks>\d+?)w)?(?:(?P<days>\d+?)d)?(?:(?P<hours>\d+?)h)?$', value)
|
|
if not match:
|
|
self.fail('%s is not a valid age' % value)
|
|
|
|
return timedelta(**{k: int(v) for k, v in match.groupdict(0).items()})
|
|
|
|
|
|
AGE = AgeParamType()
|
|
|
|
PROVIDER = click.Choice(sorted(provider_manager.names()))
|
|
|
|
REFINER = click.Choice(sorted(refiner_manager.names()))
|
|
|
|
dirs = AppDirs('subliminal')
|
|
cache_file = 'subliminal.dbm'
|
|
config_file = 'config.ini'
|
|
|
|
|
|
@click.group(context_settings={'max_content_width': 100}, epilog='Suggestions and bug reports are greatly appreciated: '
|
|
'https://github.com/Diaoul/subliminal/')
|
|
@click.option('--addic7ed', type=click.STRING, nargs=2, metavar='USERNAME PASSWORD', help='Addic7ed configuration.')
|
|
@click.option('--legendastv', type=click.STRING, nargs=2, metavar='USERNAME PASSWORD', help='LegendasTV configuration.')
|
|
@click.option('--opensubtitles', type=click.STRING, nargs=2, metavar='USERNAME PASSWORD',
|
|
help='OpenSubtitles configuration.')
|
|
@click.option('--omdb', type=click.STRING, nargs=1, metavar='APIKEY', help='OMDB API key.')
|
|
@click.option('--cache-dir', type=click.Path(writable=True, file_okay=False), default=dirs.user_cache_dir,
|
|
show_default=True, expose_value=True, help='Path to the cache directory.')
|
|
@click.option('--debug', is_flag=True, help='Print useful information for debugging subliminal and for reporting bugs.')
|
|
@click.version_option(__version__)
|
|
@click.pass_context
|
|
def subliminal(ctx, addic7ed, legendastv, opensubtitles, omdb, cache_dir, debug):
|
|
"""Subtitles, faster than your thoughts."""
|
|
# create cache directory
|
|
try:
|
|
os.makedirs(cache_dir)
|
|
except OSError:
|
|
if not os.path.isdir(cache_dir):
|
|
raise
|
|
|
|
# configure cache
|
|
region.configure('dogpile.cache.dbm', expiration_time=timedelta(days=30),
|
|
arguments={'filename': os.path.join(cache_dir, cache_file), 'lock_factory': MutexLock})
|
|
|
|
# configure logging
|
|
if debug:
|
|
handler = logging.StreamHandler()
|
|
handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
|
|
logging.getLogger('subliminal').addHandler(handler)
|
|
logging.getLogger('subliminal').setLevel(logging.DEBUG)
|
|
|
|
ctx.obj = {
|
|
'provider_configs': {},
|
|
'refiner_configs': {}
|
|
}
|
|
|
|
# provider configs
|
|
if addic7ed:
|
|
ctx.obj['provider_configs']['addic7ed'] = {'username': addic7ed[0], 'password': addic7ed[1]}
|
|
if legendastv:
|
|
ctx.obj['provider_configs']['legendastv'] = {'username': legendastv[0], 'password': legendastv[1]}
|
|
if opensubtitles:
|
|
ctx.obj['provider_configs']['opensubtitles'] = {'username': opensubtitles[0], 'password': opensubtitles[1]}
|
|
ctx.obj['provider_configs']['opensubtitlesvip'] = {'username': opensubtitles[0], 'password': opensubtitles[1]}
|
|
|
|
# refiner configs
|
|
if omdb:
|
|
ctx.obj['refiner_configs']['omdb'] = {'apikey': omdb}
|
|
|
|
|
|
@subliminal.command()
|
|
@click.option('--clear-subliminal', is_flag=True, help='Clear subliminal\'s cache. Use this ONLY if your cache is '
|
|
'corrupted or if you experience issues.')
|
|
@click.pass_context
|
|
def cache(ctx, clear_subliminal):
|
|
"""Cache management."""
|
|
if clear_subliminal:
|
|
for file in glob.glob(os.path.join(ctx.parent.params['cache_dir'], cache_file) + '*'):
|
|
os.remove(file)
|
|
click.echo('Subliminal\'s cache cleared.')
|
|
else:
|
|
click.echo('Nothing done.')
|
|
|
|
|
|
@subliminal.command()
|
|
@click.option('-l', '--language', type=LANGUAGE, required=True, multiple=True, help='Language as IETF code, '
|
|
'e.g. en, pt-BR (can be used multiple times).')
|
|
@click.option('-p', '--provider', type=PROVIDER, multiple=True, help='Provider to use (can be used multiple times).')
|
|
@click.option('-r', '--refiner', type=REFINER, multiple=True, help='Refiner to use (can be used multiple times).')
|
|
@click.option('-a', '--age', type=AGE, help='Filter videos newer than AGE, e.g. 12h, 1w2d.')
|
|
@click.option('-d', '--directory', type=click.STRING, metavar='DIR', help='Directory where to save subtitles, '
|
|
'default is next to the video file.')
|
|
@click.option('-e', '--encoding', type=click.STRING, metavar='ENC', help='Subtitle file encoding, default is to '
|
|
'preserve original encoding.')
|
|
@click.option('-s', '--single', is_flag=True, default=False, help='Save subtitle without language code in the file '
|
|
'name, i.e. use .srt extension. Do not use this unless your media player requires it.')
|
|
@click.option('-f', '--force', is_flag=True, default=False, help='Force download even if a subtitle already exist.')
|
|
@click.option('-hi', '--hearing-impaired', is_flag=True, default=False, help='Prefer hearing impaired subtitles.')
|
|
@click.option('-m', '--min-score', type=click.IntRange(0, 100), default=0, help='Minimum score for a subtitle '
|
|
'to be downloaded (0 to 100).')
|
|
@click.option('-w', '--max-workers', type=click.IntRange(1, 50), default=None, help='Maximum number of threads to use.')
|
|
@click.option('-z/-Z', '--archives/--no-archives', default=True, show_default=True, help='Scan archives for videos '
|
|
'(supported extensions: %s).' % ', '.join(ARCHIVE_EXTENSIONS))
|
|
@click.option('-v', '--verbose', count=True, help='Increase verbosity.')
|
|
@click.argument('path', type=click.Path(), required=True, nargs=-1)
|
|
@click.pass_obj
|
|
def download(obj, provider, refiner, language, age, directory, encoding, single, force, hearing_impaired, min_score,
|
|
max_workers, archives, verbose, path):
|
|
"""Download best subtitles.
|
|
|
|
PATH can be an directory containing videos, a video file path or a video file name. It can be used multiple times.
|
|
|
|
If an existing subtitle is detected (external or embedded) in the correct language, the download is skipped for
|
|
the associated video.
|
|
|
|
"""
|
|
# process parameters
|
|
language = set(language)
|
|
|
|
# scan videos
|
|
videos = []
|
|
ignored_videos = []
|
|
errored_paths = []
|
|
with click.progressbar(path, label='Collecting videos', item_show_func=lambda p: p or '') as bar:
|
|
for p in bar:
|
|
logger.debug('Collecting path %s', p)
|
|
|
|
# non-existing
|
|
if not os.path.exists(p):
|
|
try:
|
|
video = Video.fromname(p)
|
|
except:
|
|
logger.exception('Unexpected error while collecting non-existing path %s', p)
|
|
errored_paths.append(p)
|
|
continue
|
|
if not force:
|
|
video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values())
|
|
|
|
if check_video(video, languages=language, age=age, undefined=single):
|
|
refine(video, episode_refiners=refiner, movie_refiners=refiner,
|
|
refiner_configs=obj['refiner_configs'],
|
|
embedded_subtitles=not force, providers=provider, languages=language)
|
|
videos.append(video)
|
|
continue
|
|
|
|
# directories
|
|
if os.path.isdir(p):
|
|
try:
|
|
scanned_videos = scan_videos(p, age=age, archives=archives)
|
|
except:
|
|
logger.exception('Unexpected error while collecting directory path %s', p)
|
|
errored_paths.append(p)
|
|
continue
|
|
for video in scanned_videos:
|
|
if not force:
|
|
video.subtitle_languages |= set(search_external_subtitles(video.name,
|
|
directory=directory).values())
|
|
if check_video(video, languages=language, age=age, undefined=single):
|
|
refine(video, episode_refiners=refiner, movie_refiners=refiner,
|
|
refiner_configs=obj['refiner_configs'], embedded_subtitles=not force,
|
|
providers=provider, languages=language)
|
|
videos.append(video)
|
|
else:
|
|
ignored_videos.append(video)
|
|
continue
|
|
|
|
# other inputs
|
|
try:
|
|
video = scan_video(p)
|
|
except:
|
|
logger.exception('Unexpected error while collecting path %s', p)
|
|
errored_paths.append(p)
|
|
continue
|
|
if not force:
|
|
video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values())
|
|
if check_video(video, languages=language, age=age, undefined=single):
|
|
refine(video, episode_refiners=refiner, movie_refiners=refiner,
|
|
refiner_configs=obj['refiner_configs'], embedded_subtitles=not force,
|
|
providers=provider, languages=language)
|
|
videos.append(video)
|
|
else:
|
|
ignored_videos.append(video)
|
|
|
|
# output errored paths
|
|
if verbose > 0:
|
|
for p in errored_paths:
|
|
click.secho('%s errored' % p, fg='red')
|
|
|
|
# output ignored videos
|
|
if verbose > 1:
|
|
for video in ignored_videos:
|
|
click.secho('%s ignored - subtitles: %s / age: %d day%s' % (
|
|
os.path.split(video.name)[1],
|
|
', '.join(str(s) for s in video.subtitle_languages) or 'none',
|
|
video.age.days,
|
|
's' if video.age.days > 1 else ''
|
|
), fg='yellow')
|
|
|
|
# report collected videos
|
|
click.echo('%s video%s collected / %s video%s ignored / %s error%s' % (
|
|
click.style(str(len(videos)), bold=True, fg='green' if videos else None),
|
|
's' if len(videos) > 1 else '',
|
|
click.style(str(len(ignored_videos)), bold=True, fg='yellow' if ignored_videos else None),
|
|
's' if len(ignored_videos) > 1 else '',
|
|
click.style(str(len(errored_paths)), bold=True, fg='red' if errored_paths else None),
|
|
's' if len(errored_paths) > 1 else '',
|
|
))
|
|
|
|
# exit if no video collected
|
|
if not videos:
|
|
return
|
|
|
|
# download best subtitles
|
|
downloaded_subtitles = defaultdict(list)
|
|
with AsyncProviderPool(max_workers=max_workers, providers=provider, provider_configs=obj['provider_configs']) as p:
|
|
with click.progressbar(videos, label='Downloading subtitles',
|
|
item_show_func=lambda v: os.path.split(v.name)[1] if v is not None else '') as bar:
|
|
for v in bar:
|
|
scores = get_scores(v)
|
|
subtitles = p.download_best_subtitles(p.list_subtitles(v, language - v.subtitle_languages),
|
|
v, language, min_score=scores['hash'] * min_score / 100,
|
|
hearing_impaired=hearing_impaired, only_one=single)
|
|
downloaded_subtitles[v] = subtitles
|
|
|
|
if p.discarded_providers:
|
|
click.secho('Some providers have been discarded due to unexpected errors: %s' %
|
|
', '.join(p.discarded_providers), fg='yellow')
|
|
|
|
# save subtitles
|
|
total_subtitles = 0
|
|
for v, subtitles in downloaded_subtitles.items():
|
|
saved_subtitles = save_subtitles(v, subtitles, single=single, directory=directory, encoding=encoding)
|
|
total_subtitles += len(saved_subtitles)
|
|
|
|
if verbose > 0:
|
|
click.echo('%s subtitle%s downloaded for %s' % (click.style(str(len(saved_subtitles)), bold=True),
|
|
's' if len(saved_subtitles) > 1 else '',
|
|
os.path.split(v.name)[1]))
|
|
|
|
if verbose > 1:
|
|
for s in saved_subtitles:
|
|
matches = s.get_matches(v)
|
|
score = compute_score(s, v)
|
|
|
|
# score color
|
|
score_color = None
|
|
scores = get_scores(v)
|
|
if isinstance(v, Movie):
|
|
if score < scores['title']:
|
|
score_color = 'red'
|
|
elif score < scores['title'] + scores['year'] + scores['release_group']:
|
|
score_color = 'yellow'
|
|
else:
|
|
score_color = 'green'
|
|
elif isinstance(v, Episode):
|
|
if score < scores['series'] + scores['season'] + scores['episode']:
|
|
score_color = 'red'
|
|
elif score < scores['series'] + scores['season'] + scores['episode'] + scores['release_group']:
|
|
score_color = 'yellow'
|
|
else:
|
|
score_color = 'green'
|
|
|
|
# scale score from 0 to 100 taking out preferences
|
|
scaled_score = score
|
|
if s.hearing_impaired == hearing_impaired:
|
|
scaled_score -= scores['hearing_impaired']
|
|
scaled_score *= 100 / scores['hash']
|
|
|
|
# echo some nice colored output
|
|
click.echo(' - [{score}] {language} subtitle from {provider_name} (match on {matches})'.format(
|
|
score=click.style('{:5.1f}'.format(scaled_score), fg=score_color, bold=score >= scores['hash']),
|
|
language=s.language.name if s.language.country is None else '%s (%s)' % (s.language.name,
|
|
s.language.country.name),
|
|
provider_name=s.provider_name,
|
|
matches=', '.join(sorted(matches, key=scores.get, reverse=True))
|
|
))
|
|
|
|
if verbose == 0:
|
|
click.echo('Downloaded %s subtitle%s' % (click.style(str(total_subtitles), bold=True),
|
|
's' if total_subtitles > 1 else ''))
|