diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..8eae399ce --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +.gitattributes export-ignore +/Wiki export-ignore +.gitignore export-ignore diff --git a/.gitignore b/.gitignore index 8c1667465..58b6bae7c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ build/ develop-eggs/ dist/ eggs/ -lib/ lib64/ parts/ sdist/ @@ -53,3 +52,8 @@ coverage.xml # Sphinx documentation docs/_build/ +# pycharm +.idea + +icon.psd +main-icon.psd \ No newline at end of file diff --git a/Contents/Libraries/Shared/requests/packages/urllib3/contrib/__init__.py b/.gitmodules similarity index 100% rename from Contents/Libraries/Shared/requests/packages/urllib3/contrib/__init__.py rename to .gitmodules diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100755 index 000000000..21d385b32 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,1283 @@ +2.6.5.3280 + +temporarily enable OpenSubtitles.com instead of OpenSubtitles.org. +You need to have an account there and an API consumer configured. Enter your API key in settings. + +This is barely tested but should work for basic usage. + +THIS PLUGIN IS DEPRECATED, PLEASE USE BAZARR! + +Changelog +- cheaply backport opensubtitlescom from bazarr + + +2.6.5.3268 +subscene, addic7ed +- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration + +Changelog +- clarify README +- core: fix custom folder handling; #761 +- core: providers: screwzira: move to ktuvit and wizdom +- core: add option to not download subtitles for certain audio languages existing; and/or no audio stream; fix #756 +- core: delay item refreshes after refresh call (default: 5 seconds; exposed in advanced settings) +- menu: allow extraction of embedded subtitles for whole tv shows + + +2.6.5.3247 + +subscene, addic7ed + +either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service (anti-captcha.com or deathbycaptcha.com), add funds, then supply your credentials/apikey in the configuration +Changelog +core: fix for tv.plex.agents.movie not populating its media types +core: tasks: findBetterSubtitles: increase minimum score for better subtitles for movies with extracted embedded subs from 82 to 112 + + +2.6.5.3241 + +subscene, addic7ed + +either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service (anti-captcha.com or deathbycaptcha.com), add funds, then supply your credentials/apikey in the configuration +Changelog + +core: add support for new Plex Movie agent +core: remove py3 compat breaking unnecessary change +core: skip drawing tags for SRT +core: advanced_settings: refiners: drone: add custom pem_file support; fixes #735 +providers: core: set DownloadLimitPerDayExceeded timeout to 4 hours (was one day); +providers: addic7ed: limit downloads per day; add vip setting +providers: addic7ed: properly compare last_dl, add last_reset tracking info to log #723 +providers: addic7ed: properly implement limits +submod: HI: remove more music tags +submod: common CM_punctuation_space2: detect AND don't try changing domain/url/host when fixing punctuation + + +2.6.5.3223 + +subscene, addic7ed + +either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service (anti-captcha.com or deathbycaptcha.com), add funds, then supply your credentials/apikey in the configuration +Changelog + +core: scoring: reorder subtitles based on second non-hash-score if main hash score is the same; morpheus65535/bazarr#821 +providers: bsplayer: verify hash; clean up + + +2.6.5.3217 + +subscene, addic7ed +- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration + +Changelog +- core: also extract (missing) embedded subtitles when SearchAllRecentlyAddedMissing is running +- core: core: clarify detecting streams (in logs) +- core: UnRAR: set binary to executable, even if not checked out from git; might fix #693 +- core: bazarr-backport: morpheus65535/bazarr#703: use proper language code detection instead of a wild guess; should fix bad existing subtitle detection +- core: bazarr-backport: morpheus65535/bazarr#660: fix BOM encoding stuff +- core: bazarr-backport: morpheus65535/bazarr#656 further generalize formats; skip release group match if format match failed +- core: fix stream detection when using mediainfo (#711) +- config/core: make periodic SZ-internal subtitle maintenance interval configurable +- providers: add BSPlayer Subtitles +- providers: add ScrewZira (Hebrew) + +2.6.5.3183 + + +subscene, addic7ed +- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration + +Changelog +- core: don't fall back to default providers if none enabled +- core: don't process any further if stream info is missing +- core: support using mediainfo for retrieving MP4 MOV_TEXT subtitle stream titles (PMS bug) +- core: fix embedded subtitle extraction in some cases (#681, #680) +- core: scanning: add additional INFO logging for undetected languages +- core: bazarr-backport: remove existing subtitle file, to support MergerFS +- core: bazarr-backport: generic 10 minute throttling if uncaught exception occurs +- providers: addic7ed: fix recaptcha solving; fix show ID retrieval (#681) +- providers: addic7ed: add timeout on authentication error +- providers: addic7ed: fix shows with dots in them (Mayans M.C.) +- providers: addic7ed: fix detection of completed subtitle for non-english users (#686) +- providers: addic7ed: add more timeouts in the login process +- providers: argenteam: bazarr-backport: use new url; fixes + + +2.6.5.3152 + +subscene, addic7ed +- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration + +Changelog +- core: fix core issue possibly impacting results on OpenSubtitles in certain conditions +- core: fix default values of opensubtitles-skip-wrong-fps, use_https; fix #676 +- core: fix for determining whether to search under certain circumstances; fixes #666 +- core: #664 fix missing language processing of multiple videos refreshed at once +- core: #661 fix match strictness when determining preexisting external subtitles +- providers: titlovi: New implementation of Titlovi using API (thanks @viking1304) + + +2.6.5.3124 + +subscene, addic7ed and titlovi +- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration + +Changelog +- core: http: fallback to default DNS when normal resolving fails; fixes #657 +- core: extract embedded/menu: fix detection of unknown streams; don't use unknown streams if a known language was previously found +- core: language: use replacement map from bazarr +- providers: titlovi: fix matching +- providers: subscene: fix unknown language code error when "empty" result is returned +- providers: subscene: add support for pt-BR (based on Diaoul/subliminal@b22cf08) +- providers: subscene: explicitly set account filters for languages +- providers: subscene: limit alternative searches to 3; set throttle to 8 +- providers: subscene: move login/cookies to initialization sequence +- submod: generic: en: fix ";='s + + +2.6.5.3109 + +subscene, addic7ed and titlovi +- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration + +Changelog +- providers: add Napisy24 (polish) +- providers: subscene: reduce provider load by possibly half +- providers: subscene: support logging in (username/password are now required) +- providers: subscene: fallback to non year results if none found with year + + +2.6.5.3099 + +subscene, addic7ed and titlovi +- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration + +Changelog +- core: allow system DNS again by putting "system" as the DNS +- providers: subscene: fix again (subscene, contact us please, so we can end this) + + +2.6.5.3092 + +subscene, addic7ed and titlovi +- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration + +Changelog +- providers: subscene: fix endpoint (hopefully for longer now) +- providers: subscene: don't search for season packs (broken for now; relieves 50% of server load on provider) +- providers: subscene: don't calculate video fn for now +- providers: argenteam: backport fixes from bazarr +- subtitle: try decoding with utf-16 by default as well (zho/farsi) +- submod: HI: remove music tags by default +- core: compat (bazarr): add env var SZ_KEEP_ENCODING to keep encoding of subtitles + + +2.6.5.3074 + +Changelog +- core: cf: bypass cf 95% of the time without captchas +- core: fix breaking line endings of certain languages (chinese, UTF-16); fixes #646 +- core: update pysubs2 to 0.2.3 + + +2.6.5.3062 + +Changelog +- core: cf: optimize +- core: http: don't query DNS with IPs. thanks @fgump (fixes sonarr/radarr) + + +2.6.5.3041 + +Changelog +- core: only reference guessed title if there actually is one +- core: cf: optimize +- core/config: add setting for one existing language to be enough, fixes #491 +- core/compat: dns: support nameservers via ENV[dns_resolvers]; don't fall back to default DNS when configured custom DNS failed +- providers: titlovi: prevent repeated captcha solving for CF + + +2.6.5.3017 + +Changelog +- core: SRT parsing: handle (bad) ASS color tag in SRT +- core: auto extract embedded: only use one unknown sub for first language +- core: better embedded streams language detection +- core: optimizations +- core: extract embedded: fix is_unknown check +- core: don't raise exception when subtitle not found inside archive +- core: search external subtitles: fix condition +- core: better plex transcoder path detection +- core: use Log.Warn instead of Log.Warning (#619, #629, #633) +- core: also check for "plex transcoder.exe" in case of windows (fixes #619) +- core: auto extract: use mbcs encoding for paths on windows +- core: Fix issue scandir not returning the name of the file inside Docker images on ARM systems. (thanks @giejay) +- core: also clean PYTHONHOME when calling external notification app +- core: update certifi to 2019.3.9 +- core: scan_video: add series/title as alternative by scanning filename itself without parent folders +- core: add generic solution for solving captchas using anti captcha services +- core: increase cache time to 180d (was: 30d) +- core: guess_matches: handle multiple title matches; fixes bazarr#403 +- windows: fix compatibility issues with plex transcoder +- compat: use lowercase paths on subtitle detection +- providers: addic7ed: re-enable (using paid anti captch service) +- providers: assrt: assume undefined Chinese flavor as Simplified (chs/zho-Hans) +- providers: subscene: make it work again by bypassing cf +- providers: subscene: don't fail on missing cover +- providers: titlovi: re-enable (might need paid anti captch service) +- providers: opensubtitles: fix only_foreign handling +- providers: opensubtitles: show subtitles with possibly mismatched series when manually listing subs +- menu: list subtitles: show subtitles with bad season/episode values as well +- refiners: omdb: fix imdb ids with spaces + + +2.6.4.2911 +- core: improve file cache (windows especially); use fixed-length cache filenames; fixes #600 +- core: don't log "Checking connections ..." when sonarr/radarr not activated +- core: make logging for scanning/parsing/preparing videos more clear +- core: extract embedded: continue searching for embbedded subs after undefined language is found (thanks @jippo015) +- compat: core: don't assume hints["title"] exists +- compat: providers: podnapisi: loosen lxml requirement +- providers: addic7ed: fix not using user credentials; fixes #605 +- providers: titlovi: fix provider (thanks @viking1304) +- providers: subscene: fix provider +- providers: opensubtitles: improve token logging +- providers: podnapisi: fix searching for Marvel series +- submod: HI: correctly remove uppercase at start of a sentence when lead by a crocodile (>>) +- submod: HI: correctly remove lowercase inside brackets when HI matched as well +- submod: HI: remove multiple HI_before_colon_caps before one colon +- submod: common: also match music symbols after a crocodile; move crocodile removal up the chain + + +2.6.4.2881 +- core: extract embedded: fix automatic extraction not actually writing the subtitles to disk under certain circumstances; fixes #598 +- core: sonarr/radarr: don't block the main thread while checking connectivity; fixes #597 +- providers: hosszupuska: fix inconsistent series naming (thanks @morpheus133) +- providers: opensubtitles: add advanced setting to optionally not skip subtitles with wrong FPS; fixes #578 + + +2.6.4.2864 +- core: scanning: don't fail on metadata subtitles with bad language code; fixes #596 +- providers: legendastv, napiprojekt, subscenter, tvsubtitles: fix "No language to search for" issue; fixes #596 +- menu: fix "ignore list list" +- menu: advanced: add skip next search all recently missing subtitles entry + + +2.6.4.2859 +- core: fix thread.lock error (only affected the history menu, not the actual functionality) +- core: fix audio-based conditional subtitle decision making; fixes #592 +- core: massively improve metadata subtitle storage +- providers: opensubtitles: skip non-forced results when searching for forced +- providers: podnapisi: skip non-forced results when searching for forced +- submod: common: correctly pad music symbols on either side + + + +2.6.4.2834 +- core: add option to use custom (Google, Cloudflare) DNS to resolve provider hosts in problematic countries; fixes #547 +- core: add support for downloading subtitles only when the audio streams don't match (any?) configured languages; fixes #519 +- core: add support for an include list instead of an ignore list; add the option to disable SZ by default, then enable it per item/series/section (inverse ignore list) +- core/menu/config: support forced/foreign subtitles independently +- core: fallback for OSError on scandir, should fix #532 +- core: add config versioning/migration system +- core: correctly force non-foreign-only-capable providers off; remove subscene from foreign-only capable providers +- core: scanning: collect information about audio streams +- core: use correct storage path when storing subtitle info, when only VTT is used +- core: fix disabled channel mode +- core/menu: extract embedded: add extracted embedded subtitles to history +- core: embedded subtitle streams: don't try parsing the language if inexistant +- core: subtitle: fix log call, fixes #569 +- core: download best subtitles: only use actually languages searched for +- core: refiners: tvdb: warn instead of error when no matching series was found +- core: scanning: re-add expected title to guessit for narrowing down the video title +- core: resolve #583 +- core: archives: explicitly skip forced subtitles if not searched for, when picking from an archive +- core: activities/auto-refresh: fix hybrid-plus for movies +- core: don't disable plugin if all providers throttled; fix #585 #574 +- core: skip cleanup for ignored paths +- core: update requests to 2.20.0 (fixes security issue) +- core: update certifi to 2018.10.15 +- core: auto extract: don't overwrite local sub even if unknown to SZ +- config: set autoclean leftover/unused to off by default +- providers: opensubtitles: respect rate limit (40 hits/10s); should fix long throttling behaviour +- providers: opensubtitles: handle bad/inexistant responses +- providers: opensubtitles: log bad response data +- providers: opensubtitles: treat empty response as ServiceUnavailable for now +- providers: opensubtitles: log reason for ServiceUnavailable +- providers: legendastv: match second title and imdb id +- providers: titlovi: fix language handling (thanks @viking1304) +- providers: titlovi: proper handling of archives with both cyrlic and latin subtitles (thanks @viking1304) +- providers: titlovi: allow direct subtitle downloads as fallback (when a subtitle, not an archive was returned) +- providers: hosszupuska: implement site change (thanks @morpheus133) +- providers: supersubtitles: add base properties to subtitle +- providers: opensubtitles, podnapisi: fix foreign/forced handling +- providers: subscene: use original/sceneName if possible +- menu: fix plugin not responding when ignoring an item in certain menus; fixes #535 +- menu: select active subtitle: return to item details afterwards; correctly set current +- menu: add item thumbnails to history and a couple of submenus +- menu: history: use series thumbnail instead of episode screenshot +- menu: add full soft include/exclude menu handling +- menu: add support for separate forced and not-forced subtitles +- menu: fix order of embedded subtitle streams in item detail +- menu: support S00E00 and equivalent +- submod: add option to fix only-uppercase subtitles and make them readable +- submod: keep track of actually applied mods +- submod: correctly merge mods of the same kind (offset) +- submod: OCR: add dictionaries for bosnian and norwegian bokmal; update dicts for dan, eng, hrv, spa, srp, swe +- submod: OCR/HI: skip certain processors for all-caps subs +- submod: HI: only remove caps before colon if the colon is followed by whitespace or EOL; fixes #542 +- submod: HI: remove MAN: +- submod: common: improve detection and normalization of quotes, apostrophes +- submod: common: fix double quotes that are meant to be single quotes inside words +- submod: common: normalize small hyphens to dash +- submod: common: remove line only consisting of colon; remove empty colon at start of line +- submod: common: add space after punctuation +- submod: common: fix lowercase i for english language +- submod: common: better fix for music symbols +- submod: reverse_RTL: also reverse ":,'-" chars in CM_RTL_reverse (thanks @doopler) +- submod: reverse_RTL: enable mod for arabic, farsi and persian besides hebrew +- i18n: fix not used translation for recently added missing subtitles menu +- i18n: fix spanish translation, fixes #543 +- i18n: Hungarian translation is incomplete + + + +2.5.7.2663 +- implement translations for the channel and the settings +- i18n: German (myself) +- i18n: Danish (thanks Uthman, Claus Møller, dane22) +- i18n: Dutch (thanks jippo015, Semi Doludizgin, Rafael) +- i18n: Hungarian (thanks Morpheus1333, sugarman402) +- i18n: Spanish LA&C (thanks Yamil.llanos, Notorius28) +- core: notify executable: support spaces in path, fixes #520 +- core: notify executable: fix usage with python scripts (drops inherited PYTHONPATH), fixes #355 +- core: fix plugin_pin_mode +- refiners: filebot: fix usage on OSX +- providers: add assrt.net (Chinese) - thanks @dimotsai! +- providers: add supersubtitles (feliratok.info, Hungarian) - thanks @morpheus133! +- providers: addic7ed: cache login data instead of re-login per search +- submod: HI: support "&" and "+" in hi_before_colon +- submod: HI: be less aggressive with HI_before_colon_noncaps; fixes #510 + + +2.5.4.2541 + +- core: try retrieving advanced_settings.json from the path given, which may be a file path or a directory +- menu: ignore options: fix plugin not responding, fix unicode strings; resolve #509 +- providers: addic7ed: fix usage/adapt to new show search method +- providers: opensubtitles: properly handle responses again, re-enable automatic throttling based on those (broken since XMLRPC handler rewrite) + + +2.5.4.2527 + +- core: bugfixes +- core: get_item: don't fail on socket timeout; fixes #498 +- core: fix scandir encoding errors; #453 #461 #441 +- core: clamp menu history to 25 items +- add UnRAR for aarch64 (untested), arm (armv5tel, untested), linux/i386, MacOSX/i386; fixes #311 +- add 3rd party licenses +- menu: new debounce/history mechanism; fixes the back button usage +- config: add custom path option for advanced_settings.json +- providers: opensubtitles: re-add support for throttling based on HTTP response codes, which got ditched due to new connection interface +- providers: legendastv: disable if unrar wasn't found +- providers: addic7ed: reduce show cache to 1 week +- advanced settings: sonarr/radarr: make ssl verification optional +- advanced settings: opensubtitles: add configurable connection timeout +- refiners: drone: use certifi for HTTPS connections +- tasks: SearchAllRecentlyAddedMissing: fix ZeroDivisionError in edgecases; fixes #496 + + +2.5.3.2452 + +- core: update certifi to 2018.01.18 +- core: metadata storage: only allow one subtitle per language +- core: metadata storage: only parse latest metadata subtitle in localmedia +- core: metadata storage: kill existing metadata subtitles explicitly upon storing a new one +- core: metadata storage: fix selecting current subtitle from menu +- providers: opensubtitles: use new requests based transport by default, finally fixes ResponseNotReady properly +- providers: opensubtitles: mask token in logs +- providers: don't check for hash validity if it isn't verifiable (fixes napiprojekt, #478) +- submod: common: extend non_word_only matching +- submod: common: reduce multi spaces to one +- submod: OCR: fix III'll=I'll +- advanced settings: add option to use HTTP instead of HTTPS for OpenSubtitles + + +2.5.3.2422 + +- core: don't fail on embedded subtitle streams without language code set, fixes #473 +- providers: catch ResponseNotReady in list_subtitles_provider as well (partly fixes OpenSubtitles) +- providers: don't use retry logic in case of ResponseNotReady +- providers: addic7ed: use new search endpoint + + +2.5.3.2414 + +- core: expand user agent list +- core: update subliminal to 4ad5d31 +- core: treat 23.976, 23.98, 24.0 fps as equal +- core: correctly skip blacklist entries when iterating through currently known subs +- core: fix unpacking of packs without asked-for-release-group +- core: fix embedded subtitle language detection; add debug log +- core: treat embedded subtitle containing "forced" in its title as forced +- core: improve embedded subtitles detection +- core: store extracted embedded forced subtitles with the "forced" suffix (e.g.: video.en.forced.srt) +- core: don't bother trying to extract embedded subtitle if transcoder wasn't found +- core: fix automatic extraction of unknown embedded subtitle streams +- core: skip immediately searching for new subtitle after successfully extracting embedded +- core: extract embedded ASS: don't transcode to SRT using ffmpeg (Plex Transcoder), do the transcoding later using pysubs2; fixes offset issues +- core: extract embedded: let ffmpeg auto convert mov_text/tx3g to srt +- core: fix transcoder detection; add fallback #460 +- core: remove LD_LIBRARY_PATH from environment before calling notification executable +- core: auto extract embedded subtitles in a separate thread +- core: reduce encoding change log spam +- core: only allow one automatic extraction at a time; add optional advanced settings "auto_extract_multithread" +- core: add minimum score a subtitle has to have when considered by the find better subtitles task, when the current subtitle is an extracted embedded one; add advanced_settings entries +- core/config: automatic extraction: add config setting to indicate whether there should be an immediate search for available subtitles after extraction or not (default: off) +- core/menu/submod: add reverse_rtl modification for Hebrew; fixes #409 +- core: scoring: assume title match on tvdb_id match +- tasks: search all recently added missing: fix attribute access on missing stored subtitle info +- providers: add hosszupuska (hungarian, thanks morpheus133 for the basic implementation) +- providers: add argenteam (spanish, thanks mmiraglia for the basic implementation) +- providers: addic7ed: use random user agent by default (enforce for existing configs) +- providers: enable subscene by default +- providers: opensubtitles: add fallback for dict based query response in contrast to list/array based +- advanced settings: make text-based-subtitle-formats configurable +- menu: submod: inverse-reverse subtitle timing time-choices for better accessibility +- submod: reduce log spam in case of debug logs enabled +- submod: style tags could result in no output at all +- submod: fix empty content if only non-line-mods were used, no line-mods; fixes #449 +- submod: HI: correctly handle style tags when checking for brackets +- submod: HI: don't remove anything that's surrounded by quotes +- submod: HI: double or triple dash is em dash +- submod: HI: HI_before_colon_noncaps, don't assume single quotes are sentence enders +- submod: common: don't uppercase after abbreviations +- submod: common: don't break phone numbers (more than one spaced number pair found) +- submod: common: also count lines only consisting of dots as removable +- submod: common: replace more than 3 consecutive dots with 3 dots +- submod: OCR: "H i." = "Hi." + + +2.5.0.2287 + +- core: reduce main icon size +- core: fix usage on NVIDIA SHIELD (hopefully, please report back), #441 +- core: add scandir fallback to listdir in case of badly configured locale in environment, #441, #440 +- core: get subtitles from archive: don't assume an episode match +- core: get subtitles from archive: don't assume any attributes in guess +- core: improve release group detection for drone/filebot/file_info refiners +- core: fix language detection for embedded subtitle streams +- core: support extraction of embedded mov_text subtitles in mp4 video files +- refiners: drone: add http:// to url if not given +- providers: opensubtitles: retry/reinitialize request when encountering ResponseNotReady +- config: clarify subscene being only enabled for TV series by default +- menu: when encountering permission errors when scanning media files, warn in the menu about them +- submod: common: don't break -- addic7ed -- +- submod: common: remove lines that consist only of dash, underscore +- submod: OCR: fix Ls = Is +- submod: OCR: fix bad HI colons (ANNOUNCER; instead of ANNOUNCER:) +- submod: common: fix lines consisting only of bad music symbols (*#¶ = ♪) +- submod: HI: remove music-symbol-only-lines +- submod: HI: be less aggressive about lines ending with a colon; please re-apply all your mods via advanced menu +- submod: OCR: fix it'sjust, isn'tjust, Iam, Ican + + +2.5.0.2247 +- fix ignoring by-hash-matched episodes + + +2.5.0.2241 + +- fix issue when removing crap from filenames to not accidentally remove release group #436 +- fix initialization of soft ignore list after upgrade fron 2.0 + + +2.5.0.2221 + +- refiners: add support for retrieving original filename from + - drone derivates: sonarr, radarr + - filebot + - symlinks + - file_info meta file lists (see wiki) + +- providers: add subscene (disabled by default to not flood subscene on release) + - normal search + - season pack search if season has concluded + +- core: add provider subtitle-archive/pack cache for retrieving single subtitles from previously downloaded (season-) packs (subscene) +- core/agent: massive performance improvements over 2.0 +- core/agent/background-tasks: reduce memory usage to a fraction of 2.0 +- core/providers: add dynamic provider throttling when certain events occur (ServiceUnavailable, too many downloads, ...), to lighten the provider-load +- core/agent/config: automatically extract embedded subtitles (and use them if no current subtitle) +- core: fix internal subtitle info storage issues +- core: always store internal subtitle information even if no subtitle was downloaded (fixes SearchAllRecentlyAddedMissing) +- core: fix internal subtitle info storage on windows (gzip handling is broken there) +- core: don't fail on missing logfile paths +- core: fix default encoding order for non-script-serbian +- core: improve logging +- core: add AsRequested to cleanup garbage names +- core: treat SDTV and HDTV the same when searching for subtitles +- core: parse_video: trust PMS season and episode numbers +- core: parse_video: add series year information from PMS if none found +- core: upgrade dependencies +- core: update subliminal to 62cdb3c +- core: add new file based cache mechanism, rendering DBM/memory backends obsolete +- core: treat 23.980 fps as 23.976 and vice-versa +- core: add HTTP proxy support for querying the providers (supports credentials) +- core: only compute file hashes for enabled providers +- core: massive speedup; refine only when needed, exit early otherwise +- core: store last modified timestamp in subtitle info storage +- core: only write to subtitle info storage if we haven't had one or any subtitle was downloaded +- core: only clean up the sub-folder if a subtitle-sub-folder has been selected, and not the parent one also +- core: support for CP437 encoded filenames in ZIP-Archives +- core: use scandir library instead of os.listdir if possible, reducing performance-impact +- core: archives: support multi-episode subtitles (partly) +- core: subtitle cleanup: add support for hi, cc, sdh secondary filename tags; don't autoclean .txt +- core: increase request timeout by three times in case a proxy is being used +- core: fix language=Unknown in Plex when "Restrict to one language"-setting is set +- core: refining: re-add old detected title as alternative title after re-refining with plex metadata's title; fixes #428 +- core: implement advanced_settings.json (see advanced_settings.json.template for reference, copy to "Plug-in Support/Data/com.plexapp.agents.subzero" to use it) +- core/tasks: fix search all recently added missing (the total number of items will change in the menu while running), reduces memory usage +- core/menu: add support for extracting embedded subtitles using the builtin plex transcoder +- core/menu: skip wrong season or episode in returned subtitle results +- core/config: fix language handling if treat undefined as first language is set +- providers: remove shooter.cn +- providers: add support for zip/rar archives containing more than one subtitle file +- submod: common: remove redundant interpunction ("Hello !!!" -> "Hello!") +- submod: skip provider hashing when applying mods +- submod: correctly drop empty line (fixing broken display) +- submod: OCR: fix F'xxxxx -> Fxxxxx +- submod: HI: improve bracket matching +- submod: OCR: fix l/L instead of I more aggressively +- submod: common: fix uppercase I's in lowercase words more aggressively +- submod: HI: improve HI_before_colon +- submod: common: be more aggressive when fixing numbers; correctly space out spaced ellipses; don't break spaced ellipses; handle multiple spaces in numbers +- menu: add support for extracting embedded subtitles for a whole season +- menu: add reapply mods to current subtitle +- menu: pad titles for more submenus, resulting in detail view in PlexWeb +- menu: add subtitle selection submenu (if multiple subtitles are inside the subtitle info storage; e.g. previously downloaded ones or extracted embedded) +- menu: advanced: add skip findbettersubtitles menu item, which sets the last_run to now (for debugging purposes) +- menu: ignore: add more natural title for seasons and episodes (kills your old ignore lists!) +- config: skip provider hashing on low impact mode +- config: add limit by air date setting to consider for FindBetterSubtitles task (default: 1 year) +- advanced settings: define enabled-for media types per provider +- advanced settings: define enabled-for languages per provider +- advanced settings: add deep-clean option (clean up the subtitle-sub-folder and the parent one) + + + +2.0.33.1871 +- core: normalize line endings in subtitles to LF (\n) +- core: add subtitle storage lock to avoid race condition +- core: be more verbose about subtitle storage addition +- core: fix MPL2 newline parsing, which resulted in broken subtitles +- core: encoding change: reduce log spam +- submod: common: fix CM_starting_spacedots +- opensubtitles: fix request/response handling + + + +2.0.33.1849 +- opensubtitles: add VIP server handling + preference; VIP benefits: 10€/year, ad-free subs, 1000 subs/day, no-cache VIP server, help SZ and subscribe via http://v.ht/osvip +- opensubtitles: try to reuse previous token instead of logging in every time +- core: add throttling between searches (10 seconds) +- core: fix IETF handling for good +- core: fix no subtitles being searched in certain situations (when an external subtitle without special tag exists) +- core: add subtitle blacklist +- core: fixes +- core: fix detection of certain PMS media stream language tags ("FR" for example) +- core: missing subtitles: correctly skip unwanted subtitle extensions +- core: missing subtitles: honor "treat undefined as first language" option correctly +- api: add blacklisting endpoints for quickly searching for new subtitls via bookmarklet +- submod: colors: apply color mods at the end of processing modifications; fix color mods +- submod: new remove_tags modification to remove all styling tags from subtitles +- submod: HI: be more aggressive at handling brackets +- submod: OCR: update en and hrv +- submod: common: remove "torrent downloaded from ..." lines +- submod: OCR: fix WholeWord handling, improving modification +- submod: apply OCR fixes before HI +- submod: OCR: fix broken HI tag colons (ANNOUNCER'. instead of ANNOUNCER:) +- menu: advanced: speed up batch modifications +- menu: add subtitle blacklist +- menu: recently played: show only TV episodes and movies (music tracks were listed here as well) + + +2.0.29.1767 +- core: fix internal subtitle storage issues +- core: handle "embedded-forced" tag (futureproofing) +- core: remove more garbage tags from release groups (nzbgeek, chamele0n, buymore, xpost, postbot) +- submod: OCR fix: fix music icon = paragraph + + +2.0.29.1756 +- core: don't fail on uppercase file extensions +- core: don't re-download a subtitle if we already downloaded one, it still physically exists and external subtitles are configured to be ignored +- core: fix VTT subtitle duplication +- core: if forced subtitles not explicitly wanted, ignore existing forced subtitles when searching +- core: add full IETF language support for `Treat languages with country attribute as ISO 639-1 (e.g. don't download pt-BR if pt subtitle exists)`-setting for embedded subtitles +- menu: remove buggy dynamic permission-based channel icon introduced in 1715 +- menu: improve `Items with missing subtitles` menu usage and item display +- menu: `Advanced -> Get my logs` handle custom domains without port +- menu: correctly show country/script part of languages with such attributes (e.g. pt-BR) +- config: rename `Scan:` settings; make them better understandable and translatable +- config: rephrase IETF options as "languages with country attribute" (e.g. pt-BR) +- config: separate IETF options into how to display languages with country attribute and how they should be handled when searching/scanning (e.g. pt-BR) +- config: `Scheduler: Item age to be considered recent` now can go up to 12 weeks +- config: `Scheduler: Periodically search for recent items with missing subtitles` added `every 2 hours` +- submod: swe: add Ĺ to Å + + +2.0.26.1715 +- core: submod: OCR fixes: swe: replace ĺ with å inside words +- core: fix handling of non-existant PMS audio_codec info +- core: filename matching ignored the strictness setting in certain global directory configurations (thanks @raduc) +- core: don't fail on migration errors +- provider titlovi: handle multiple subtitles per archive +- provider addic7ed: reset default boost to 19 (was 21) +- menu: add warning icon on missing permissions +- menu: manual subtitle list sometimes listed duplicates (thanks @andreashoyer) +- menu: don't request PMS metadata in item details menu twice +- menu: don't fail badly on non existant PMS metadata in item details menu + + +2.0.26.1695 +## ATTENTION: THIS RELEASE RESETS YOUR CONFIGURED LANGUAGES TO DEFAULT! +- core: fix bug that caused SZ not to work for Windows users with special characters in their username +- core: fix issues when logging failed manual download actions +- core: update guessit to 2.1.4 +- core: fix issue causing the background task scheduler to stop after changing preferences +- core: fix polish encoding (try windows-1250 first, then iso 8859-2) +- core: remove subscenter provider as it now uses captchas +- core: add titlovi as default provider (thanks viking!) +- core: increase default PMS API request timeout to 15 (old: 10, max: 45); add preference for that +- core: re-add separate legacy FindMissingSubtitles task and run it on the first run to prime SZ's internal subtitle storage +- core: add "low impact mode" for people with remote filesystems (currently enabled for List LANGUAGE subtitles in detail menu); alleviates certain plexweb timeout issues +- menu: change naming of find missing subtitles menu item +- legendastv: fix multi value guessit issues +- submod: OCR: update eng and hrv OCR replace dictionaries; fix ". L am huge" + + +2.0.25.1635 +- core: update memory handling, possibly reduce memory problems of 2.0 +- core: support for MPL2 subtitle format +- core: update task handling +- core: re-enable NVIDIA SHIELD support by fixing rarfile behaviour +- core: add SZ_UNRAR_TOOL environment variable for custom unrar location +- core: disable SZ when no providers are enabled +- core: only start activity monitor if channel or agent are enabled +- core: improve custom provider integration +- core: update eastern european encoding detection (especially Romanian) +- tasks: reduce provider stress by introducing wait times between searches/downloads +- windows: correctly ship UnRAR.exe +- windows: skip DBM checks +- addic7ed: fix Nip/Tuck +- subscenter: use new domain + + +2.0.24.1581 +- legendastv: ship unrar.exe for Windows users (fixes unrar issues) +- addic7ed: fix TooManyRequests error +- submod: OCR fixes NL: add custom dictionary data for malformed characters +- submod: OCR fixes: update hrv/NL dictionaries +- submod: common: remove spaces before punctuation +- podnapisi: now returns more subtitles again +ATTENTION: Sub-Zero is still broken on PMS for SHIELD. Help needed! + + +2.0.24.1565 +- core: fix searchallrecentlymissing task erroring if item not found +- core: fix non-plex-items appearing in and crashing the recently played list +- core: add hybrid-plus activity setting (current media file and next episode) +- podnapisi: fix by using correct guessit parameters + + +2.0.24.1558 +- core: fix handling of broken RAR files from legendas + + +2.0.24.1555 +- core: fix rare microdvd issue from OpenSubtitles by generally providing FPS info when encountering a microdvd subtitle + + +2.0.24.1549 +Changes from 1.4 +- wiki: new wiki! (thanks @dane22!) +- core: update subliminal to version 2 +- core: update all dependencies +- core: add new providers: legendastv (pt-BR), napiprojekt (pl), shooter (cn), subscenter (heb) +- core: rewritten all subliminal patches for version 2 +- core: use SSL again for opensubtitles +- core: improved matching due to subliminal 2 (and SZ custom) tvdb/omdb refiners +- core: improved matching by relying on existing metadata provided by the PMS +- core: improved performance due to multithreaded provider-querying +- core: improved performance due to less physical media file access (no more MKV metadata scanning) +- core: VTT subtitle format output supported (for Chromecast) +- core: rewrote and streamlined internal subtitle data storage format +- core: support Cyrillic and Latin variants of Serbian language +- core: simplified (custom) provider registration; add own provider registry +- core: rewrote recently added missing task +- core: automatically fix badly (re-) encoded unicode entities in subtitles +- core: always store subtitles in proper UTF-8 encoding +- core: add periodic internal subtitle data storage cleanup task +- core: on non-windows systems, utilize a file-based cache database for provider media lists and subliminal refiner results +- core: add manual and automatic subtitle modification framework (fix common OCR issues, remove hearing impaired etc.) +- core: relieve some stress on providers by providing better fine-grained retry handling +- menu: add icons for menu items; update main channel icon +- menu: add subtitle modifications (subtitle content fixes, offset-based shifting, framerate conversion) +- menu: add recently played menu +- menu: add "Get my logs" function to the advanced menu, which zips up all necessary logs suitable for posting in the forums +- menu: add generic "back to season" and "back to series" entries to item detail views to make navigation easier +- config: all scores changed (defaults updated) +- config: remove "Force UTF-8 when storing subtitles" (it's now always implied) +- improve almost everything Sub-Zero did in 1.4 :) + + +2.0.23.1464 RC10.1 + - core: huge bugfix; please check `Library/Application Support/Plex Media\ Server/Plug-in Support/Data/com.plexapp.agents.subzero/DataItems` + for any `subs_XXXXX.json.gz` file bigger than 500kb and delete them + + +2.0.23.1456 RC10 +- core: findBetterSubtitles: increase series cutoff by 2 (resolution match) +- core: add VTT format +- core: fix crashes regarding DBM/cache management +- core: update rarfile.py +- core: add missing encodings +- core: full support for Serbian subtitles (Cyrillic and Latin) +- podnapisi: fix pt-BR, srp-cyrl and srp-latn +- core: implement own provider registry and ditch the subliminal one +- core: use ftfy library to fix re-encoding errors inside subtitles introduced by the subtitle author +- core: always store and save subtitles normalized to UTF-8 +- core: replace spaced dashes in movie/series names before re-refining with plex metadata info +- submod: remove_HI: handle multiline brackets correctly + + +2.0.20.1364 RC9 +- core: performance improvements +- core: if info couldn't be guessed from the filename, fill missing info from PMS #270 +- submod: OCR: add more to the eng dictionary +- submod: HI: fixed some issues with font style tags +- core: don't ignore subtitles from providers that don't have hearing impaired info, when hearing impaired mode is set to "force non-HI" +- legendastv/menu: fix manual subtitle selection issues in menu +- core: improve specials matching on OpenSubtitles +- core: update guessit + + +2.0.19.1337 RC8 +- napiprojekt: fixed: couldn't convert microdvd to SRT in certain occasions +- core: when normalize to UTF-8 is enabled, also store the subtitle in UTF-8 encoding in the internal storage +- core: add more encodings for western/eastern/northern europe +- submod: OCR: update dictionaries from SubtitleEdit +- submod: common: be smarter about uppercase i's in words that should have lowercase L's +- submod: fix unopened/unclosed font style tags after modification +- core: re-enable OMDB support +- core: update guessit for better matching +- core: fix SearchAllRecentlyMissing (was broken since RC3) + + +2.0.19.1299 RC7 +- submod: offset mods now get merged internally when applied multiple times (to avoid errors and increase performance) +- submod: improve performance +- submod: core mods (OCR, common, remove_HI) now are always applied in a fixed order internally, regardless of the order they were added in +- submod: CM_spaces_in_numbers: don't break up ellipses (30... 29... 28...) +- submod: CM_spaces_in_numbers: don't fix countdown numbers (30, 29, 28) +- submod: remove_HI: make bracket removal more aggressive +- submod: remove_HI: be less aggressive when removing text-before-colon +- submod: remove_HI: remove all-uppercase-before-sentence (THIS IS ALL UPPERCASE And here starts a sentence -> And here starts a sentence) +- submod: fix all character ranges to include non-ASCII characters +- add new README for 2.0 + + +2.0.19.1267 RC6 +- core: add new SZ subtitle storage format + - smaller data files and less cumbersome + - it will auto migrate when old data is accessed - to speed this up, use "Trigger subtitle storage migration (expensive)" in advanced menu) +- core: performance optimizations +- addic7ed: when release group matches, assume the format matches, too (leftover change from RC5) +- submod: fix patterns for beginlines/endlines +- submod: add our own dictionaries to OCR fixes (english) +- submod: hearing impaired: also remove full-caps with punctuation inside +- submod: correctly handle partiallines +- submod: in numbers with spaces (incorrect), also allow for some punctuation (,.:') + + +2.0.18.1245 RC5 +- core: add more debug info +- core: fix subtitle modifications (was broken in RC4, created non-usable subtitles) +- submod: add ANSI colors +- menu/submod: add color mod menu +- submod: exclusive mods now are mutually exclusive and get cleaned on duplicate +- menu/core: naming + +For everyone who runs RC4: your subtitles are broken. Go to the advanced menu and trigger `Re-Apply mods of all stored subtitles` to fix them. + + +2.0.17.1234 RC4 +- core: backport provider-download-retry implementation +- core: implement custom user agent (for OpenSubtitles) +- core/menu: correct handling of media with multiple files +- core: fix SearchAllRecentlyMissing; also wait 5 seconds between searches +- core: SearchAllRecentlyMissing: honor physical ignores +- submod: pattern fixes +- submod: better unicode handling +- submod: add color mod (only automatic by now) + + +2.0.15.1216 RC3 +- core: fixes +- scheduler: revert some of the aggressive changes in RC2 +- submod: be smarter about WholeLine matches + + +2.0.15.1209 RC2 +- core: fixes +- core: submod-common: fix multiple dots at start of line +- core/menu: add subtitle modification debug setting +- core/menu: when manually listing available subtitles in menu, display those with wrong FPS also (opensubtitles), because you can fix them later +- core/menu: advanced-menu: add apply-all-default-mods menu item; add re-apply all mods menu item +- core: always look for currently (not-) existing subtitles when called; hopefully fixes #276 +- scheduler/menu: be faster; also launch scheduled tasks in threads, not just manually launched ones +- core: don't delete subtitles with .custom or .embedded in their filenames when running auto cleanup, if the correct media file exists +- menu: add back-to-previous menu items + + +2.0.12.1180 RC1 +- core: update subliminal to version 2 +- core: update all dependencies +- core: add new providers: legendastv (pt-BR), napiprojekt (pl), shooter (cn), subscenter (heb) +- core: rewritten all subliminal patches for version 2 +- menu: add icons for menu items; update main channel icon +- core: use SSL again for opensubtitles +- core: improved matching due to subliminal 2 (and SZ custom) tvdb/omdb refiners +- menu: add "Get my logs" function to the advanced menu, which zips up all necessary logs suitable for posting in the forums +- core: on non-windows systems, utilize a file-based cache database for provider media lists and subliminal refiner results +- core: add manual and automatic subtitle modification framework (fix common OCR issues, remove hearing impaired etc.) +- menu: add subtitle modifications (subtitle content fixes, offset-based shifting, framerate conversion) +- menu: add recently played menu +- improve almost everything Sub-Zero did in 1.4 :) + + +1.4.27.973 +- core: ignore "obfuscated" and "scrambled" tags in filenames when searching for subtitles +- core: exotic embedded subtitles are now also considered when searching (and when the option is enabled); fixes #264 + + +1.4.27.967 +- core: remember the last 10 played items; only consider on_playback for "playing" state within the first 60 seconds of an item + + +1.4.27.965 +- core: on_playback activity bugfixes + + +1.4.27.957 +- core: correctly fall back to the next best subtitle if the current one couldn't be downloaded; hopefully fixes #231 +- core: add "Scan: which external subtitles should be picked up?"-setting +- core: add optional on_playing activities. refresh currently playing movie, refresh next episode in season, both or none; fixes #259 #33 +- core: skip to next best subtitle if findbettersubtitles failed +- core: add setting to treat undefined-language embedded subtitle as configured language1 #239 +- core: fix handling of inexistant addic7ed show id +- core: fix regression issue breaking relative custom subtitle folder handling +- core: fix loading of stored subtitle info data of now-non-existant items +- core: re-add separate global subtitle folder handling +- menu: remove obsolete actions from the advanced menu + + +1.4.23.920 +- core: handle undecodable paths better #255 +- core: don't fail on unrecoverable data #257 +- core: increase default scores from 110 (series) and 23 (movies) to 116 and 33 +- core: fix global subtitle folder handling #234 +- core: better invoking of configured executable after subtitle addition #247 + + +1.4.22.908 +- core: hotfix for more robust migrations + + +1.4.22.898 +- core: migrate history and subtitle storage to a better implementation, making it far more stable. subtitle storage now also stores the downloaded subtitle data for future usage, so it will be possible to switch between them +- core/menu: manual subtitle download and the FindBetterSubtitles-task now also work with metadata storage (hi @ shield users) +- core: optimize FindBetterSubtitles-task + + +1.4.19.882 +- core: fix tasks for new users +- core: double check pin correctness/existance when pin is enabled + + +1.4.19.878 +- core/menu: fix a task's last runtime display +- core: task optimizations +- core: fix leftover subtitles cleanup handling in case of a custom subtitle folder #234 +- core: run the scheduler even if permissions for libraries are wrong ("fixes" #236) +- core: store subtitle history data in a different data format; reduce used storage size drastically (#233) + + +1.4.19.866 +- core: fix wrong usage of LogKit + + +1.4.19.857 + +- core: add option to enable/disable channel and/or agent modes (fixes #220) +- core: skip inexistent internal streams when scanning for internal subtitles (fixes #222) +- core: fix filename encoding (fixes #223) +- core: storage optimizations +- menu: add pin-based channel menu locking (the whole channel or only the advanced menu) + + +1.4.17.836 +- core: support for any media file that PMS supports (internal subtitles on mp4 for example) +- core: fix broken ignore folders containing "subzero.ignore/.subzero.ignore/.nosz" +- core: fix duplicate subtitles (lowercase/default case) +- core: fix broken tasks queue due to oversight + + +1.4.16.822 +- menu: add per-section recently added menu +- menu: fix accidentally double-triggering a just triggered force-refresh +- core: reorder settings in a more logical, grouped way +- core: add simple automatic filesystem/external leftover subtitle cleaning (#133, #152) +- core: fix force-refresh for big seasons/series +- core: add setting to look for forced/foreign-only subtitles only (only works for opensubtitles and podnapisi) +- core: fix custom subtitle folder was being ignored (#211) +- core: only trust PMS for its movie name, not the series title (fixes #210) +- core: full support (in filesystem/external mode) for forced/default/normal subtitle tags +- core: ignore "non-standard" external subtitle files when scanning by default (everything but .srt, .ass, .ssa, fixes #192) +- core: lower default max_recent_items_per_library to 500 +- core: skip forced/foreign-only subtitles if not specifically wanted +- core: modify the task queue, hopefully helping #206 +- core: update anonymous usage collection + + +1.4.11.781 +- core: cleanup, logging +- core/menu: fix addic7ed display in manual subtitle list +- core: use HTTP for OpenSubtitles instead of HTTPS because of current certificate errors +- core: find better subtitles should now run smoothly even with replaced files (newer parts) + + +1.4.10.769 +- core: hotfix for legacy intent storage regression + +1.4.10.768 +- core: automatically find better subtitles (configurable) +- menu: display how the subtitle was downloaded (auto, manual, auto-better), in history menu +- menu/core: correctly handle subtitle list for multiple languages +- core: lower minimum series score to list subtitles for to 66 +- core: better matching of garbage filenames; we trust Plex now for the series name/movie title fully +- core: add setting to specifically set the file permissions (chmod) + + +1.4.5.742 +- core: fix force-refresh in certain situations +- menu: add history +- menu: add manual subtitle selection +- menu: run Items with missing subtitles in separate thread for big libraries +- settings: add history list size option (default: 100) +- settings: add new default scores (TV: 110); use input instead of dropdown +- settings: increase default missing subtitles amount per library to 2000 +- core: generic rewrites and optimizations +- core: better hash verification +- core: add anonymous usage data (opt-out in settings) +- core: fix pt-BR display (IETF) again +- wiki: update (thanks @dane22!) - quick URL: http://v.ht/szwiki +- wiki: add score explanation - quick URL: http://v.ht/szscores +- core: add persian/farsi encoding support + + +1.3.49.636 +- core/menu: fix force refreshing (again) +- core/menu: fix redundant route calls + + +1.3.49.630 (backported some changes of the develop-1.4 branch to 1.3) +- core/menu: make addic7ed boost configurable; lower the default boost value massively (to 10) +- core: fix force refreshing (hopefully) +- core: add (thai) tis-620 subtitle encoding support +- menu: lower letter based menu browsing from 200 to 80 items +- core: support greek encodings (windows-1253, cp1253, cp737, iso8859_7, cp875, cp869, iso2022_jp_2, mac_greek); hopefully fixes badly saved greek subs +- menu: add generic back-to-home button to the top of every container view +- menu: warn the user when SZ isn't enabled for any sections/libraries +- menu: always re-check permissions status and enabled sections when opening the main menu; no server restart necessary anymore + + +1.3.46.606 +- core: hotfix for new users (who've never downloaded a subtitle with SZ before); fixes #169 + + +1.3.46.605 + +- add wiki (thanks @ukdtom / @dane22) +- core: remove necessity of Plex credentials; fixes #148 +- core: fix non-SRT subtitle support; fixes #138 +- core: generic source overhaul in preparation for release 1.4 +- core: better filesystem encoding detection; may fix #159 +- core: add encoding handling for windows-1250 and windows-1251 encoding (eastern europe); fixes #162 +- core: overhaul ignore handling; fixes #164 +- core: implement ignore by path setting; fixes #134 +- core: add setting for optional fallback to metadata storage, if filesystem storage failed; fixes #100 +- core: add setting for notifying an executable after a subtitle has been downloaded (see Wiki); fixes #65 +- core: only handle sections for which Sub-Zero is enabled (in PMS agent settings); fixes #167 +- menu: add series/season force-refresh +- menu: show item thumbnail/art where applicable +- menu: mitigate PlexWeb behaviour of calling our handlers twice; fixes #168 + + +1.3.33.522 + +- core: fix library permission detection on windows; fixes #151 +- core: "Restrict to one language" now behaves like it should (one found subtitle of any language is treated as sufficient); fixes #149 +- core: add support for other subtitle formats such as ssa/ass/microdvd, convert to srt; fixes #138 +- core: hopefully more consistent force-refresh handling (intent); fixes #118 + + +1.3.31.513 + +- core: add option to only download one language again (and skip the addition of .lang to the subtitle filename) (default: off); fixes #126 +- core: add option to always encode saved subtitles to UTF-8 (default: on); fixes #128 +- core: add fallback encoding detection using bs4.UnicodeDammit; hopefully fixes #101 +- core: update libraries: chardet, beautifulsoup, six +- menu/core: check Plex libraries for permission problems on plugin start and report them in the channel menu (option, default: on); fixes #143 +- menu: while a manual refresh takes place, add a refresh button to the top of the SZ menu for convenience +- menu: move the "add/remove X to ignore list" menu item to the bottom of the list on item detail + + +1.3.27.491 + +- menu/core: make Sub-Zero channel menu optional (setting: "Enable Sub-Zero channel (disabling doesn't affect the subtitle features)?") +- OpenSubtitles: detect and match video/subtitle FPS (framerate) to reduce out of sync subtitle matches +- core: internal fixes; add _markerlib library (rare) +- core: don't score tvshow episode title matches, should improve episode subtitle matches quite a bit (and reduce out of sync subtitles) +- OpenSubtitles: make tag/exact filename matches optional (setting: "I keep the exact (release-) filename of my media files") +- menu: unicode video title errors fixed +- TVSubtitles: correctly match certain show IDs (such as "Series Name (US)") +- core: don't break subtitle evaluation on crashed guessing + + +1.3.23.459 + +- core: slight code cleanup and fixes +- core: add physical (filesystem) ignore mode (create files named `subzero.ignore`, `.subzero.ignore`, `.nosz` to ignore specific files/seasons/series/libraries) +- core: fix guessit hinting of tv series with rare folder layout (e.g. series_name/a/S01E01.mkv) +- core: remove "format" necessity from (opensubtitles) hash-validation +- OpenSubtitles: dramatically improve matching: add tag (exact filename) matching and treat it just like hash matches +- core: ignore embedded forced subtitles (fixes #106) +- docs: update +- settings: clarify + + +1.3.20.422 +- tvsubtitles: show matching was partially broken +- addic7ed: better show matching +- core: correctly skip subtitles stored in filesystem if metadata storage was selected (Local Media Assets agent may still pick them up) +- core: fix local API access (switch from HTTPS to HTTP) +- core: fix handling of library names and media paths with non-ascii chars in it +- core: fix bundle version to correctly display current bundle version +- core: skip downloading multi-CD subtitle +- settings: clarify + + +1.3.20.403 +- core: handle & and - ("and" and dash) in names +- core: fixed handling of internal metadata subtitles +- re-upped the minimum tv score to 85 (may be even higher in the future) +- opensubtitles: possibly significantly better movie matching (now also query for movie title, instead of only querying for video hash) + + +1.3.20.396 +- core: fix logging handlers (when saving log_level settings loggers got duplicated) +- core: better movie matching by only hinting the filename and the last subdirectory to guessit (instead of the full path) +- core: don't fail on wrong detection/scanning of media file +- lower minimum tv series score from 85 to 67 (removed title; composed of: series=44 + season=11 + episode=11 + hearing_impaired=1) + + +1.3.19.379 +- core: new recent items implementation (used in "Items with missing subtitles"), now really picking up everything instead of using Plex's recently_added API endpoint +- core: be more strict about title matching - a matched title doesn't automatically mean season and episode are correct, too +- core: rewrote the hash matching algorithm to not blindly trust hash matches anymore, but instead episodes have to match the series name, season number, episode number and format (BluRay, HDTV...); movie have to at least match the title, format and codec for the hash to be considered +- core: remove TheSubDB support for now, as it only supports hash-based matching +- scheduler: more robust item-fail-handling (fixes #81) +- config: "Scan: include embedded subtitles" now by default is off, as embedded subs have proven to be pretty unreliable +- config: add configuration option for how many items per library are to be considered recent (default: 200) +- config: make logging verbosity configurable, default: WARNING - log files should be considerably smaller now +- config: make console logging optional, default: off - good for development/debugging +- config: removed the ignore lists +- menu: added "Browse all items", where you can browse all your libraries and manage your ignore list (add/remove sections/series/items) +- menu: added "Display ignore list", where you can manage your ignored sections, series and items +- menu: the submenu titles are now dynamically composed of a breadcrumb-style tree so you see where you are +- menu: show the current and past state of the important menu actions such as (force)-refresh an item or refreshing the menu, on the Refresh-button's description +- plugin now isn't in the dev mode by default and has logging to the console off (in certain configurations this resulted in huge syslogs) + + +1.3.6.316 +- scheduler: missing subtitles task now able to handle huge libraries (thanks @chopeta, @comrade) +- scheduler: detect item-stalling, add wait and retry logic to make missing subtitles task more robust +- scheduler: report failed items to logs after task run completion +- hint series name and episode title, or movie title to guessit to make detection way better (e.g. for Mr. Robot) + +1.3.6.304 +- scheduler: correct the recent-determination of the search for missing subtitles in recently_added task +- scheduler: rewrote search for missing subtitles task; it now requests refreshes one by one and not in bulk anymore (hopefully fixes stalling) +- handle rare cases of weird file system encodings (ANSI_X3.4-1968 for example) +- fix simplejson warning on startup + +1.3.6.297 +- rename Sub-Zero to Sub-Zero.bundle (requirement for adding Sub-Zero to the Plex channel directory) +- channel: add logging actions for the internal storage to the advanced menu +- channel: handle item titles with foreign characters in them correctly +- (hopefully) fix handling file names with foreign characters in them when scanning for local media +- reformat the whole project, mostly honoring pep8 +- scheduler: fixed some serious bugs; broken tasks (stalled) and some errors many of you have seen should be gone now +- scheduler: partly rewritten to be more robust, again +- settings: move Plex.tv credentials to the top + +1.3.5.281 +- fix tasks broken for 1.2 -> 1.3.5 upgraders + +1.3.5.273 (same build as Beta Release 1.3.0.273) - changes from previous stable 1.2.11.180 +- add a channel menu, making this plugin a hybrid (Agent+Channel) +- add a generic background task scheduler +- add a task to search for subtitles for items with missing subtitles (manually triggered and automatic) +- add artwork +- add Plex.tv credentials/token-generation support (needed for Plex Home users for the API to work) +- addic7ed: improve show name matching again +- channel: able to browse current on-deck and recently-added items, and refresh or force-refresh (search for new subtitles) single items +- add library/series/video blacklist for items which should be skipped in "Search for missing subtitles"-task +- add donation links +- change the license to The Unlicense (while keeping the original MIT license from subliminal.bundle intact) +- store subtitle information in internal plugin storage (for later usage) +- many internal code improvements +- update documentation + +1.3.0.273 +- more robust update functionality +- menu: add refresh button to menu (to see the task state updating) +- scheduler: actually skip a task if it's already running +- scheduler: better behaviour when a task is running and a single item is refreshed at the same time +- menu: enforce ascii on item titles + +1.3.0.261 +- removed localization again + +1.3.0.259 +- forgot locale-data + +1.3.0.256 +- fix force-refresh single items to actually force-refresh +- re-add babel library + +1.3.0.253 +- rewrote background tasks subsystem +- keep track of the status of a task and its runtime +- add task state in channel menu to "Search for missing subtitles" +- add date/time localization to channel menu +- hide plex token from logs, when requesting +- fix addic7ed show id parsing for shows with year set +- test PMS API connectivity and fail miserably if needed (channel disabled, scheduler disabled) +- feature-freeze for 1.3.0 final + +1.3.0.245 +- add the option to buy me a beer +- clarify menu items +- more robust scheduler handling (should fix the issues of scheduler runs in the past) +- internal cleanups +- add date_added to stored subtitle info (all of the 1.3.0 testers: please delete your internal subtitle storage using the channel->advanced menu) + +1.3.0.232 +- integrate plex.tv authentication for plex home users (test phase) +- menu cleanup +- more info in the menu (scheduler last and next run for example) +- hopefully fixed intent handling (should throw less errors now) +- fix version display in agent names + +1.3.0.222 +- bugfix for search missing subtitles +- schedduler: honor "never" + +1.3.0.216 +- add channel menu +- add generic task scheduler +- add functionality to search for missing subtitles (via recently added items) +- add artwork +- change license to The Unlicense +- ... + +1.2.11.180 +- fix #49 (metadata storage didn't work) +- add better detection for existing subtitles stored in metadata + +1.2.11.177 +- updated naming scheme to reflect rewrite.major.minor.build (this release is the same as 1.1.0.5) + +1.1.0.5 +- addic7ed: fixed error in show id search +- addic7ed: even better show matching +- adjusted default scores: TV: 85, movies: 23 +- add support for com.plexapp.agents.xbmcnfo/xbmcnfotv (proposed to the author [here](https://github.com/gboudreau/XBMCnfoMoviesImporter.bundle/pull/63) and [here](https://github.com/gboudreau/XBMCnfoTVImporter.bundle/pull/70)) + +1.1.0.3 +- addic7ed/tvsubtitles: be way smarter about punctuation in series names (*A.G.E.N.T.S. ...*) +- ditch LocalMediaExtended and incorporate the functionality in Sub-Zero (**RC-users: delete LocalMediaExtended.bundle and re-enable LocalMedia!**) +- remove (unused) setting "Restrict to one language" +- add "Treat IETF language tags as ISO 639-1 (e.g. pt-BR = pt)" setting (default: true) +- change default external storage to "current folder" instead of "/subs" +- adjust default scores + +RC-5.2 +- revert back to /plexinc-agents/LocalMedia.bundle/tree/dist instead of /plexinc-agents/LocalMedia.bundle/tree/master, as the current public PMS version is too old for that + +RC-5.1 +- make hearing_impaired option more configurable and clear (see #configuration-) + +RC-5 +- fix wrong video type matching by hinting video type to guessit +- update to newest LocalMediaExtended.bundle (incorporated plex-inc's changes) +- show page links for subtitles in log file instead of subtitle ID +- add custom language setting in addition to the three hardcoded ones +- if a subtitle doesn't match our hearing_impaired setting, ignore it +- add an optional boost for addic7ed subtitles, if their series, season, episode, year, and format (e.g. WEB-DL) matches + +RC-4 +- rename project to Sub-Zero +- incorporate LocalMediaExtended.bundle +- making this a multi-bundle plugin +- update default scores +- add icon + +RC-3 +- addic7ed/tvsubtitles: punctuation fixes (correctly get show ids for series like "Mr. Poopster" now) +- podnapisi: fix logging +- opensubtitles: add login credentials (for VIPs) +- add retry functionality to retry failed subtitle downloads, including configurable amount of retries until discarding of provider +- move possibly not needed setting "Restrict to one language" to the bottom +- more detailed logging +- some cleanup + +RC-2 +- fix empty custom subtitle folder creation +- fix detection of existing embedded subtitles (switch to https://github.com/tonswieb/enzyme) +- better logging +- set default TV score to 15; movie score to 30 + +RC-1 +- fix subliminal's logging error on min_score not met (fixes #15) +- separated tv and movies subtitle scores settings (fixes #16) +- add option to save only one subtitle per video (skipping the ".lang." naming scheme plex supports) (fixes #3) + +beta5 +- fix storing subtitles besides the actual video file, not subfolder (fixes #14) +- "custom folder" setting now always used if given (properly overrides "subtitle folder" setting) +- also scan (custom) given subtitle folders for existing subtitles instead of redownloading them on every refresh (fixes #9, #2) + +beta4 +- ~~increased score of addic7ed subtitles a bit~~ (not existing currently) +- **support for newest Subliminal ([1.0.1](27a6e51cd36ffb2910cd9a7add6d797a2c6469b7)) and guessit ([0.11.0](2814f57e8999dcc31575619f076c0c1a63ce78f2))** +- **plugin now also [works with com.plexapp.agents.thetvdbdvdorder](924470d2c0db3a71529278bce4b7247eaf2f85b8)** +- providers fixed for subliminal 1.0.1 ([at least addic7ed](131504e7eed8b3400c457fbe49beea3b115bc916)) +- providers [don't simply fail and get excluded on non-detected language](1a779020792e0201ad689eefbf5a126155e89c97) +- support for addic7ed languages: [French (Canadian)](b11a051c233fd72033f0c3b5a8c1965260e7e19f) +- support for additional languages: [pt-br (Portuguese (Brasil)), fa (Persian (Farsi))](131504e7eed8b3400c457fbe49beea3b115bc916) +- support for [three (two optional) subtitle languages](e543c927cf49c264eaece36640c99d67a99c7da2) +- optionally use [random user agent for addic7ed provider](83ace14faf75fbd75313f0ceda9b78161895fbcf) (should not be needed) diff --git a/Contents/Code/__init__.py b/Contents/Code/__init__.py old mode 100644 new mode 100755 index 3ab3f6821..1ed0daf2f --- a/Contents/Code/__init__.py +++ b/Contents/Code/__init__.py @@ -1,149 +1,275 @@ -# hdbits.org +# coding=utf-8 +import sys +import datetime + +from subzero.sandbox import fix_environment_stuff + +module = sys.modules['__main__'] +fix_environment_stuff(module, {}) + +globals = getattr(module, "__builtins__")["globals"] +for key, value in getattr(module, "__builtins__").iteritems(): + if key != "globals": + globals()[key] = value -import string, os, urllib, zipfile, re, copy -from babelfish import Language -from datetime import timedelta -import subliminal import logger -OS_PLEX_USERAGENT = 'plexapp.com v9.0' +sys.modules["logger"] = logger + +import support + +import interface +sys.modules["interface"] = interface + +from subzero.constants import OS_PLEX_USERAGENT +from interface.menu import * +from support.plex_media import media_to_videos, get_media_item_ids +from support.extract import agent_extract_embedded +from support.scanning import scan_videos +from support.storage import save_subtitles, store_subtitle_info +from support.items import is_wanted +from support.config import config +from support.lib import get_intent +from support.helpers import track_usage, get_title_for_video_metadata, get_identifier, cast_bool +from support.history import get_history +from support.data import dispatch_migrate +from support.activities import activity +from support.download import download_best_subtitles +from support.localmedia import find_subtitles -DEPENDENCY_MODULE_NAMES = ['subliminal', 'enzyme', 'guessit', 'requests'] def Start(): HTTP.CacheTime = 0 HTTP.Headers['User-agent'] = OS_PLEX_USERAGENT - Log.Debug("START CALLED") - logger.registerLoggingHander(DEPENDENCY_MODULE_NAMES) - # configured cache to be in memory as per https://github.com/Diaoul/subliminal/issues/303 - subliminal.cache_region.configure('dogpile.cache.memory') - -def ValidatePrefs(): - Log.Debug("Validate Prefs called.") - return - -# Prepare a list of languages we want subs for -def getLangList(): - langList = {Language.fromietf(Prefs["langPref1"])} - if(Prefs["langPref2"] != "None"): - langList.update({Language.fromietf(Prefs["langPref2"])}) - - return langList - -def getProviders(): - providers = {'opensubtitles' : Prefs['provider.opensubtitles.enabled'], - 'thesubdb' : Prefs['provider.thesubdb.enabled'], - 'podnapisi' : Prefs['provider.podnapisi.enabled'], - 'addic7ed' : Prefs['provider.addic7ed.enabled'], - 'tvsubtitles' : Prefs['provider.tvsubtitles.enabled'] - } - return filter(lambda prov: providers[prov], providers) - -def getProviderSettings(): - provider_settings = {'addic7ed': {'username': Prefs['provider.addic7ed.username'], - 'password': Prefs['provider.addic7ed.password'] - } - } - return provider_settings - -def scanTvMedia(media): - videos = {} - for season in media.seasons: - for episode in media.seasons[season].episodes: - for item in media.seasons[season].episodes[episode].items: - for part in item.parts: - scannedVideo = scanVideo(part) - videos[scannedVideo] = part - return videos - -def scanMovieMedia(media): - videos = {} - for item in media.items: - for part in item.parts: - scannedVideo = scanVideo(part) - videos[scannedVideo] = part - return videos - -def scanVideo(part): - embedded_subtitles = Prefs['subtitles.scan.embedded'] - external_subtitles = Prefs['subtitles.scan.external'] - - Log.Debug("Scanning video: %s, subtitles=%s, embedded_subtitles=%s" % (part.file, external_subtitles, embedded_subtitles)) - try: - return subliminal.video.scan_video(part.file, subtitles=external_subtitles, embedded_subtitles=embedded_subtitles) - except ValueError: - Log.Warn("File could not be guessed by subliminal") - -def downloadBestSubtitles(videos): - min_score = int(Prefs['subtitles.search.minimumScore']) - hearing_impaired = Prefs['subtitles.search.hearingImpaired'] - Log.Debug("Download best subtitles using settings: min_score: %s, hearing_impaired: %s" %(min_score, hearing_impaired)) - return subliminal.api.download_best_subtitles(videos, getLangList(), getProviders(), getProviderSettings(), min_score, hearing_impaired) - -def saveSubtitles(videos, subtitles): - if Prefs['subtitles.save.filesystem']: - Log.Debug("Saving subtitles to filesystem") - saveSubtitlesToFile(subtitles) - else: - Log.Debug("Saving subtitles as metadata") - saveSubtitlesToMetadata(videos, subtitles) - -def saveSubtitlesToFile(subtitles): - fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if bool(Prefs["subtitles.save.subFolder.Custom"]) else None - if Prefs["subtitles.save.subFolder"] != "current folder" or fld_custom: - # specific subFolder requested, create it if it doesn't exist - for video, video_subtitles in subtitles.items(): - fld_base = os.path.split(video.name)[0] - if fld_custom: - if fld_custom.startswith("/"): - # absolute folder - fld = fld_custom - else: - fld = os.path.join(fld_base, fld_custom) - else: - fld = os.path.join(fld_base, Prefs["subtitles.save.subFolder"]) - if not os.path.exists(fld): - os.makedirs(fld) - subliminal.api.save_subtitles({video:video_subtitles}, directory=fld) - - else: - subliminal.api.save_subtitles(subtitles) - -def saveSubtitlesToMetadata(videos, subtitles): - for video, video_subtitles in subtitles.items(): - mediaPart = videos[video] - for subtitle in video_subtitles: - mediaPart.subtitles[Locale.Language.Match(subtitle.language.alpha2)][subtitle.page_link] = Proxy.Media(subtitle.content, ext="srt") - -class SubliminalSubtitlesAgentMovies(Agent.Movies): - name = 'Subliminal Movie Subtitles' - languages = [Locale.Language.English] - primary_provider = False - contributes_to = ['com.plexapp.agents.imdb'] - def search(self, results, media, lang): - Log.Debug("MOVIE SEARCH CALLED") - results.Append(MetadataSearchResult(id='null', score=100)) + config.init_cache() - def update(self, metadata, media, lang): - Log.Debug("MOVIE UPDATE CALLED") - videos = scanMovieMedia(media) - subtitles = downloadBestSubtitles(videos.keys()) - saveSubtitles(videos, subtitles) - -class SubliminalSubtitlesAgentTvShows(Agent.TV_Shows): - - name = 'Subliminal TV Subtitles' + # clear expired intents + intent = get_intent() + intent.cleanup() + + #Locale.DefaultLocale = "de" + + # clear expired menu history items + now = datetime.datetime.now() + if "menu_history" in Dict: + for key, timeout in Dict["menu_history"].copy().items(): + if now > timeout: + try: + del Dict["menu_history"][key] + except: + pass + + # run migrations + if "subs" in Dict or "history" in Dict: + Thread.Create(dispatch_migrate) + + # clear old task data + scheduler.clear_task_data() + + # init defaults; perhaps not the best idea to use ValidatePrefs here, but we'll see + ValidatePrefs() + Log.Debug(config.full_version) + + if not config.permissions_ok: + Log.Error("Insufficient permissions on library folders:") + for title, path in config.missing_permissions: + Log.Error("Insufficient permissions on library %s, folder: %s" % (title, path)) + + # run task scheduler + scheduler.run() + + # bind activities + if config.enable_channel: + Thread.Create(activity.start) + + if "anon_id" not in Dict: + Dict["anon_id"] = get_identifier() + + # track usage + if cast_bool(Prefs["track_usage"]): + if "first_use" not in Dict: + Dict["first_use"] = datetime.datetime.utcnow() + Dict.Save() + track_usage("General", "plugin", "first_start", config.version) + track_usage("General", "plugin", "start", config.version) + + +def update_local_media(videos, ignore_parts_cleanup=None): + for video in videos: + find_subtitles(video["plex_part"], ignore_parts_cleanup=ignore_parts_cleanup) + + +class SubZeroAgent(object): + agent_type = None + agent_type_verbose = None languages = [Locale.Language.English] primary_provider = False - contributes_to = ['com.plexapp.agents.thetvdb'] + score_prefs_key = None + debounce = 10 + + def __init__(self, *args, **kwargs): + super(SubZeroAgent, self).__init__(*args, **kwargs) + self.agent_type = "movies" if isinstance(self, Agent.Movies) else "series" + self.name = "Sub-Zero Subtitles (%s, %s)" % (self.agent_type_verbose, config.get_version()) def search(self, results, media, lang): - Log.Debug("TV SEARCH CALLED") + Log.Debug("Sub-Zero %s, %s search" % (config.version, self.agent_type)) results.Append(MetadataSearchResult(id='null', score=100)) + def store_blank_subtitle_metadata(self, video_part_map): + store_subtitle_info(video_part_map, dict((k, []) for k in video_part_map.keys()), None, mode="a") + def update(self, metadata, media, lang): - Log.Debug("TvUpdate. Lang %s" % lang) - videos = scanTvMedia(media) - subtitles = downloadBestSubtitles(videos.keys()) - saveSubtitles(videos, subtitles) + if not config.enable_agent: + Log.Debug("Skipping Sub-Zero agent(s)") + return + + Log.Debug("Sub-Zero %s, %s update called" % (config.version, self.agent_type)) + + if not media: + Log.Error("Called with empty media, something is really wrong with your setup!") + return + + intent = get_intent() + + item_ids = [] + try: + config.init_subliminal_patches() + all_videos = media_to_videos(media, kind=self.agent_type) + + # media ignored? + ignore_parts_cleanup = [] + videos = [] + for video in all_videos: + if not is_wanted(video["id"], item=video["item"]): + Log.Debug(u'Skipping "%s"' % video["filename"]) + ignore_parts_cleanup.append(video["path"]) + continue + videos.append(video) + + # find local media + update_local_media(all_videos, ignore_parts_cleanup=ignore_parts_cleanup) + + if not videos: + Log.Debug(u"Nothing to do.") + return + + try: + use_score = int(Prefs[self.score_prefs_key].strip()) + except ValueError: + Log.Error("Please only put numbers into the scores setting. Exiting") + return + + set_refresh_menu_state(media, media_type=self.agent_type) + + # scanned_video_part_map = {subliminal.Video: plex_part, ...} + providers = config.get_providers(media_type=self.agent_type) + try: + scanned_video_part_map = scan_videos(videos, providers=providers) + except IOError, e: + Log.Exception("Permission error, please check your folder/file permissions. Exiting.") + if cast_bool(Prefs["check_permissions"]): + config.permissions_ok = False + config.missing_permissions = e.message + return + + # auto extract embedded + if config.embedded_auto_extract: + if config.plex_transcoder: + agent_extract_embedded(scanned_video_part_map) + else: + Log.Warn("Plex Transcoder not found, can't auto extract") + + # clear missing subtitles menu data + if not scheduler.is_task_running("MissingSubtitles"): + scheduler.clear_task_data("MissingSubtitles") + + downloaded_subtitles = None + + # debounce for self.debounce seconds + now = datetime.datetime.now() + if "last_call" in Dict: + last_call = Dict["last_call"] + if last_call + datetime.timedelta(seconds=self.debounce) > now: + wait = self.debounce - (now - last_call).seconds + if wait >= 1: + Log.Debug("Waiting %s seconds until continuing", wait) + Thread.Sleep(wait) + + # downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]} + try: + downloaded_subtitles = download_best_subtitles(scanned_video_part_map, min_score=use_score, + throttle_time=self.debounce, providers=providers) + except: + Log.Exception("Something went wrong when downloading subtitles") + + if downloaded_subtitles is not None: + Dict["last_call"] = datetime.datetime.now() + + item_ids = get_media_item_ids(media, kind=self.agent_type) + + downloaded_any = False + if downloaded_subtitles: + downloaded_any = any(downloaded_subtitles.values()) + + if downloaded_any: + save_successful = False + try: + save_successful = save_subtitles(scanned_video_part_map, downloaded_subtitles, + mods=config.default_mods) + except: + Log.Exception("Something went wrong when saving subtitles") + + track_usage("Subtitle", "refreshed", "download", 1) + + # store SZ meta info even if download wasn't successful + if not save_successful: + self.store_blank_subtitle_metadata(scanned_video_part_map) + + else: + for video, video_subtitles in downloaded_subtitles.items(): + # store item(s) in history + for subtitle in video_subtitles: + history = get_history() + item_title = get_title_for_video_metadata(video.plexapi_metadata, add_section_title=False) + history.add(item_title, video.id, section_title=video.plexapi_metadata["section"], + thumb=video.plexapi_metadata["super_thumb"], + subtitle=subtitle) + history.destroy() + else: + # store SZ meta info even if we've downloaded none + self.store_blank_subtitle_metadata(scanned_video_part_map) + + update_local_media(videos) + + finally: + # update the menu state + set_refresh_menu_state(None) + + # notify any running tasks about our finished update + for item_id in item_ids: + #scheduler.signal("updated_metadata", item_id) + + # resolve existing intent for that id + intent.resolve("force", item_id) + + Dict.Save() + + # fsync cache + if config.new_style_cache: + config.sync_cache() + + +class SubZeroSubtitlesAgentMovies(SubZeroAgent, Agent.Movies): + contributes_to = ['com.plexapp.agents.imdb', 'com.plexapp.agents.xbmcnfo', 'com.plexapp.agents.themoviedb', + 'com.plexapp.agents.hama', 'tv.plex.agents.movie'] + score_prefs_key = "subtitles.search.minimumMovieScore2" + agent_type_verbose = "Movies" + + +class SubZeroSubtitlesAgentTvShows(SubZeroAgent, Agent.TV_Shows): + contributes_to = ['com.plexapp.agents.thetvdb', 'com.plexapp.agents.themoviedb', + 'com.plexapp.agents.thetvdbdvdorder', 'com.plexapp.agents.xbmcnfotv', 'com.plexapp.agents.hama'] + score_prefs_key = "subtitles.search.minimumTVScore2" + agent_type_verbose = "TV" diff --git a/Contents/Code/interface/__init__.py b/Contents/Code/interface/__init__.py new file mode 100644 index 000000000..f59f4b0f6 --- /dev/null +++ b/Contents/Code/interface/__init__.py @@ -0,0 +1,23 @@ +import sys + +import menu +sys.modules["interface.menu"] = menu +sys.modules["menu"] = menu + +import menu_helpers +sys.modules["interface.menu_helpers"] = menu_helpers + +import advanced +sys.modules["interface.advanced"] = advanced + +import main +sys.modules["interface.main"] = main + +import refresh_item +sys.modules["interface.refresh_item"] = refresh_item + +import item_details +sys.modules["interface.item_details"] = item_details + +import sub_mod +sys.modules["interface.modification"] = sub_mod diff --git a/Contents/Code/interface/advanced.py b/Contents/Code/interface/advanced.py new file mode 100644 index 000000000..c9a25af63 --- /dev/null +++ b/Contents/Code/interface/advanced.py @@ -0,0 +1,454 @@ +# coding=utf-8 +import datetime +import StringIO +import glob +import os +import traceback +import urlparse + +from zipfile import ZipFile, ZIP_DEFLATED + +from subzero.language import Language + +from subzero.lib.io import FileIO +from subzero.constants import PREFIX, PLUGIN_IDENTIFIER +from menu_helpers import SubFolderObjectContainer, debounce, set_refresh_menu_state, ZipObject, ObjectContainer, route +from main import fatality +from support.helpers import timestamp, pad_title +from support.config import config +from support.lib import Plex +from support.storage import reset_storage, log_storage, get_subtitle_storage +from support.scheduler import scheduler +from support.items import set_mods_for_part, get_item_kind_from_rating_key +from support.i18n import _ + + +@route(PREFIX + '/advanced') +def AdvancedMenu(randomize=None, header=None, message=None): + oc = SubFolderObjectContainer( + header=header or _("Internal stuff, pay attention!"), + message=message, + no_cache=True, + no_history=True, + replace_parent=False, + title2=_("Advanced")) + + if config.lock_advanced_menu and not config.pin_correct: + oc.add(DirectoryObject( + key=Callback( + PinMenu, + randomize=timestamp(), + success_go_to=_("advanced")), + title=pad_title(_("Enter PIN")), + summary=_("The owner has restricted the access to this menu. Please enter the correct pin"), + )) + return oc + + oc.add(DirectoryObject( + key=Callback(TriggerRestart, randomize=timestamp()), + title=pad_title(_("Restart the plugin")), + )) + oc.add(DirectoryObject( + key=Callback(GetLogsLink), + title=_("Get my logs (copy the appearing link and open it in your browser, please)"), + summary=_("Copy the appearing link and open it in your browser, please"), + )) + oc.add(DirectoryObject( + key=Callback(TriggerBetterSubtitles, randomize=timestamp()), + title=pad_title(_("Trigger find better subtitles")), + )) + oc.add(DirectoryObject( + key=Callback(SkipFindBetterSubtitles, randomize=timestamp()), + title=pad_title(_("Skip next find better subtitles (sets last run to now)")), + )) + oc.add(DirectoryObject( + key=Callback(SkipRecentlyAddedMissing, randomize=timestamp()), + title=pad_title(_("Skip next find recently added with missing subtitles (sets last run to now)")), + )) + oc.add(DirectoryObject( + key=Callback(TriggerStorageMaintenance, randomize=timestamp()), + title=pad_title(_("Trigger subtitle storage maintenance")), + )) + oc.add(DirectoryObject( + key=Callback(TriggerStorageMigration, randomize=timestamp()), + title=pad_title(_("Trigger subtitle storage migration (expensive)")), + )) + oc.add(DirectoryObject( + key=Callback(TriggerCacheMaintenance, randomize=timestamp()), + title=pad_title(_("Trigger cache maintenance (refiners, providers and packs/archives)")), + )) + oc.add(DirectoryObject( + key=Callback(ApplyDefaultMods, randomize=timestamp()), + title=pad_title(_("Apply configured default subtitle mods to all (active) stored subtitles")), + )) + oc.add(DirectoryObject( + key=Callback(ReApplyMods, randomize=timestamp()), + title=pad_title(_("Re-Apply mods of all stored subtitles")), + )) + oc.add(DirectoryObject( + key=Callback(LogStorage, key="tasks", randomize=timestamp()), + title=pad_title(_("Log the plugin's scheduled tasks state storage")), + )) + oc.add(DirectoryObject( + key=Callback(LogStorage, key="ignore", randomize=timestamp()), + title=pad_title(_("Log the plugin's internal ignorelist storage")), + )) + oc.add(DirectoryObject( + key=Callback(LogStorage, key=None, randomize=timestamp()), + title=pad_title(_("Log the plugin's complete state storage")), + )) + oc.add(DirectoryObject( + key=Callback(ResetStorage, key="tasks", randomize=timestamp()), + title=pad_title(_("Reset the plugin's scheduled tasks state storage")), + )) + oc.add(DirectoryObject( + key=Callback(ResetStorage, key="ignore", randomize=timestamp()), + title=pad_title(_("Reset the plugin's internal ignorelist storage")), + )) + oc.add(DirectoryObject( + key=Callback(ResetStorage, key="menu_history", randomize=timestamp()), + title=pad_title(_("Reset the plugin's menu history storage")), + )) + oc.add(DirectoryObject( + key=Callback(InvalidateCache, randomize=timestamp()), + title=pad_title(_("Invalidate Sub-Zero metadata caches (subliminal)")), + )) + oc.add(DirectoryObject( + key=Callback(ResetProviderThrottle, randomize=timestamp()), + title=pad_title(_("Reset provider throttle states")), + )) + return oc + + +def DispatchRestart(): + Thread.CreateTimer(1.0, Restart) + + +@route(PREFIX + '/advanced/restart/trigger') +@debounce +def TriggerRestart(randomize=None): + set_refresh_menu_state(_("Restarting the plugin")) + DispatchRestart() + return fatality( + header=_("Restart triggered, please wait about 5 seconds"), + force_title=" ", + only_refresh=True, + replace_parent=True, + no_history=True, + randomize=timestamp()) + + +@route(PREFIX + '/advanced/restart/execute') +@debounce +def Restart(randomize=None): + Plex[":/plugins"].restart(PLUGIN_IDENTIFIER) + + +@route(PREFIX + '/storage/reset', sure=bool) +@debounce +def ResetStorage(key, randomize=None, sure=False): + if not sure: + oc = SubFolderObjectContainer( + no_history=True, + title1=_("Reset subtitle storage"), + title2=_("Are you sure?")) + oc.add(DirectoryObject( + key=Callback( + ResetStorage, + key=key, + sure=True, + randomize=timestamp()), + title=pad_title(_("Are you really sure?")), + + )) + return oc + + reset_storage(key) + + if key == "tasks": + # reinitialize the scheduler + scheduler.init_storage() + scheduler.setup_tasks() + + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("Information Storage (%s) reset", key) + ) + + +@route(PREFIX + '/storage/log') +def LogStorage(key, randomize=None): + log_storage(key) + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("Information Storage (%s) logged", key) + ) + + +@route(PREFIX + '/triggerbetter') +@debounce +def TriggerBetterSubtitles(randomize=None): + scheduler.dispatch_task("FindBetterSubtitles") + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("FindBetterSubtitles triggered") + ) + + +@route(PREFIX + '/skipbetter') +@debounce +def SkipFindBetterSubtitles(randomize=None): + task = scheduler.task("FindBetterSubtitles") + task.last_run = datetime.datetime.now() + + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("FindBetterSubtitles skipped") + ) + + +@route(PREFIX + '/skipram') +@debounce +def SkipRecentlyAddedMissing(randomize=None): + task = scheduler.task("SearchAllRecentlyAddedMissing") + task.last_run = datetime.datetime.now() + + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("SearchAllRecentlyAddedMissing skipped") + ) + + +@route(PREFIX + '/triggermaintenance') +@debounce +def TriggerStorageMaintenance(randomize=None): + scheduler.dispatch_task("SubtitleStorageMaintenance") + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("SubtitleStorageMaintenance triggered") + ) + + +@route(PREFIX + '/triggerstoragemigration') +@debounce +def TriggerStorageMigration(randomize=None): + scheduler.dispatch_task("MigrateSubtitleStorage") + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("MigrateSubtitleStorage triggered") + ) + + +@route(PREFIX + '/triggercachemaintenance') +@debounce +def TriggerCacheMaintenance(randomize=None): + scheduler.dispatch_task("CacheMaintenance") + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("TriggerCacheMaintenance triggered") + ) + + +def apply_default_mods(reapply_current=False, scandir_generic=False): + storage = get_subtitle_storage() + subs_applied = 0 + + try: + for fn in storage.get_all_files(scandir_generic=scandir_generic): + data = storage.load(None, filename=fn) + if data: + video_id = data.video_id + item_type = get_item_kind_from_rating_key(video_id) + if not item_type: + continue + + for part_id, part in data.parts.iteritems(): + for lang, subs in part.iteritems(): + current_sub = subs.get("current") + if not current_sub: + continue + sub = subs[current_sub] + + if not sub.content: + continue + + current_mods = sub.mods or [] + if not reapply_current: + add_mods = list(set(config.default_mods).difference(set(current_mods))) + if not add_mods: + continue + else: + if not current_mods: + continue + add_mods = [] + + try: + set_mods_for_part(video_id, part_id, Language.fromietf(lang), item_type, add_mods, mode="add") + except: + Log.Error("Couldn't set mods for %s:%s: %s", video_id, part_id, traceback.format_exc()) + continue + + subs_applied += 1 + except OSError: + return apply_default_mods(reapply_current=reapply_current, scandir_generic=True) + storage.destroy() + Log.Debug("Applied mods to %i items" % subs_applied) + + +@route(PREFIX + '/applydefaultmods') +@debounce +def ApplyDefaultMods(randomize=None): + Thread.CreateTimer(1.0, apply_default_mods) + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("This may take some time ...") + ) + + +@route(PREFIX + '/reapplyallmods') +@debounce +def ReApplyMods(randomize=None): + Thread.CreateTimer(1.0, apply_default_mods, reapply_current=True) + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("This may take some time ...") + ) + + +@route(PREFIX + '/get_logs_link') +def GetLogsLink(): + if not config.plex_token: + oc = ObjectContainer( + title2=_("Download Logs"), + no_cache=True, + no_history=True, + header=_("Sorry, feature unavailable"), + message=_("Universal Plex token not available")) + return oc + + # try getting the link base via the request in context, first, otherwise use the public ip + req_headers = Core.sandbox.context.request.headers + get_external_ip = True + link_base = "" + + if "Origin" in req_headers: + link_base = req_headers["Origin"] + Log.Debug("Using origin-based link_base") + get_external_ip = False + + elif "Referer" in req_headers: + parsed = urlparse.urlparse(req_headers["Referer"]) + link_base = "%s://%s%s" % (parsed.scheme, parsed.hostname, (":%s" % parsed.port) if parsed.port else "") + Log.Debug("Using referer-based link_base") + get_external_ip = False + + if get_external_ip or "plex.tv" in link_base: + ip = Core.networking.http_request("http://www.plexapp.com/ip.php", cacheTime=7200).content.strip() + link_base = "https://%s:32400" % ip + Log.Debug("Using ip-based fallback link_base") + + logs_link = "%s%s?X-Plex-Token=%s" % (link_base, PREFIX + '/logs', config.plex_token) + oc = ObjectContainer( + title2=logs_link, + no_cache=True, + no_history=True, + header=_("Copy this link and open this in your browser, please"), + message=logs_link) + return oc + + +@route(PREFIX + '/logs') +def DownloadLogs(): + buff = StringIO.StringIO() + zip_archive = ZipFile(buff, mode='w', compression=ZIP_DEFLATED) + + logs = sorted(glob.glob(config.plugin_log_path + '*')) + [config.server_log_path] + for path in logs: + data = StringIO.StringIO() + data.write(FileIO.read(path)) + zip_archive.writestr(os.path.basename(path), data.getvalue()) + + zip_archive.close() + + return ZipObject(buff.getvalue()) + + +@route(PREFIX + '/invalidatecache') +@debounce +def InvalidateCache(randomize=None): + from subliminal.cache import region + if config.new_style_cache: + region.backend.clear() + else: + region.invalidate() + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("Cache invalidated") + ) + + +@route(PREFIX + '/pin') +def PinMenu(pin="", randomize=None, success_go_to="channel"): + oc = ObjectContainer( + title2=_("Enter PIN number ") + str(len(pin) + 1), + no_cache=True, + no_history=True, + skip_pin_lock=True) + + if pin == config.pin: + Dict["pin_correct_time"] = datetime.datetime.now() + config.locked = False + if success_go_to == "channel": + return fatality( + force_title=_("PIN correct"), + header=_("PIN correct"), + no_history=True) + elif success_go_to == "advanced": + return AdvancedMenu(randomize=timestamp()) + + for i in range(10): + oc.add(DirectoryObject( + key=Callback( + PinMenu, + randomize=timestamp(), + pin=pin + str(i), + success_go_to=success_go_to), + title=pad_title(str(i)), + )) + oc.add(DirectoryObject( + key=Callback( + PinMenu, + randomize=timestamp(), + success_go_to=success_go_to), + title=pad_title(_("Reset")), + )) + return oc + + +@route(PREFIX + '/pin_lock') +def ClearPin(randomize=None): + Dict["pin_correct_time"] = None + config.locked = True + return fatality(force_title=_("Menu locked"), header=" ", no_history=True) + + +@route(PREFIX + '/reset_throttle') +def ResetProviderThrottle(randomize=None): + Dict["provider_throttle"] = {} + Dict.Save() + return AdvancedMenu( + randomize=timestamp(), + header=_("Success"), + message=_("Provider throttles reset") + ) diff --git a/Contents/Code/interface/func.py b/Contents/Code/interface/func.py new file mode 100644 index 000000000..d8e1e59bf --- /dev/null +++ b/Contents/Code/interface/func.py @@ -0,0 +1,185 @@ +# coding=utf-8 + +import datetime +import operator + +from support.config import config +from support.helpers import timestamp + + +def enable_channel_wrapper(func, enforce_route=False): + """ + returns the original wrapper :func: (route or handler) if applicable, else the plain to-be-wrapped function + :param func: original wrapper + :return: original wrapper or wrapped function + """ + def noop(*args, **kwargs): + def inner(*a, **k): + """ + :param a: args + :param k: kwargs + :return: originally to-be-wrapped function + """ + return a[0] if len(a) else a + + return inner + + def wrap(*args, **kwargs): + return (func if (config.enable_channel or enforce_route) else noop)(*args, **kwargs) + + return wrap + + +ROUTE_REGISTRY = {} + + +def get_func_name(args): + return list(args).pop(0).__name__ + + +def get_lookup_key(f, args, kwargs): + return tuple([f.__name__, tuple(args), tuple([(key, value) for key, value in kwargs.iteritems()])]) + + +def should_debounce(f, key, kw): + return getattr(f, "debounce", False) and "randomize" in kw and key in Dict["menu_history"] + + +def register_route_function(f): + fn = f.__name__ + if fn != "ValidatePrefs" and fn not in ROUTE_REGISTRY: + ROUTE_REGISTRY[fn] = f + return f + + +def main_menu_fallback(): + key = get_lookup_key(ROUTE_REGISTRY["fatality"], [], {}) + Dict["last_menu_item"] = key + add_to_menu_history(key) + + return ROUTE_REGISTRY["fatality"](randomize=timestamp()) + + +def add_to_menu_history(key): + # add function to menu history + mh = Dict["menu_history"] + if key in mh: + del mh[key] + + mh[key] = datetime.datetime.now() + datetime.timedelta(hours=6) + + # limit to 25 items + Dict["menu_history"] = dict(sorted(sorted(mh.items(), key=operator.itemgetter(1), + reverse=True)[:25])) + + try: + Dict.Save() + except TypeError: + Log.Error("Can't save menu history for: %r", key) + del Dict["menu_history"][key] + + +def route_wrapper(*args, **kwargs): + def wrap(f): + already_wrapped = getattr(f, "orig_f", False) + + register_route_function(f) + + def inner(*a, **kw): + if "menu_history" not in Dict: + Dict["menu_history"] = {} + + if "last_menu_item" not in Dict: + Dict["last_menu_item"] = None + + key = get_lookup_key(f, list(a), kw) + + ret_f = f + ret_a = a + ret_kw = kw + # mh = Dict["menu_history"] + # mh_keys = [k for k, v in sorted(mh.items(), key=operator.itemgetter(1))] + # + # fallback_needed = False + # fallback_found = False + + if should_debounce(ret_f, key, kw): + # special case for TriggerRestart + if ret_f.__name__ in ("TriggerRestart", "Restart"): + Log.Debug("Don't trigger a re-restart, falling back to main menu") + else: + Log.Debug("not triggering %s twice with %s, %s, returning to main menu" % + (f.__name__, a, kw)) + + return main_menu_fallback() + # + # fallback_needed = True + # + # # try to find a suitable fallback route in case we've encountered an already visited + # # debounced route + # fallbacks = [] + # current_last_visit = mh[key] + # last_menu_item = Dict["last_menu_item"] + # direction_backwards = True + # + # if last_menu_item and last_menu_item in mh and key in mh: + # last_mi_pos = mh_keys.index(last_menu_item) + # current_mi_pos = mh_keys.index(key) + # if current_mi_pos > -1 and last_mi_pos > -1: + # print "SHEKEL", current_mi_pos, last_mi_pos, current_mi_pos < last_mi_pos + + # only consider items in menu history that have an older timestamp than the current + # for key_, last_visit in sorted(mh.items(), key=operator.itemgetter(1), + # reverse=True): + # if last_visit < current_last_visit: + # fallbacks.append(key_) + # + # for key_ in fallbacks: + # # old data structure + # if not len(key_) == 3 or not (isinstance(key_[1], tuple) and isinstance(key_[2], tuple)): + # continue + # + # old_f, old_a, old_kw = key_ + # if old_f == "ValidatePrefs": + # continue + # + # possible_fallback = ROUTE_REGISTRY[old_f] + # + # # non-debounced function found + # if not getattr(possible_fallback, "debounce", False): + # ret_kw = dict(old_kw) + # ret_a = old_a + # if "randomize" in ret_kw: + # ret_kw["randomize"] = timestamp() + # + # ret_f = possible_fallback + # key = get_lookup_key(ret_f, list(ret_a), ret_kw) + # fallback_found = True + # + # Log.Debug("not triggering %s twice with %s, %s, returning to %s, %s, %s" % + # (f.__name__, a, kw, ret_f.__name__, ret_a, ret_kw)) + # + # break + # + # if not fallback_found: + # Log.Debug("No fallback found in menu history for %s, falling back to main menu", f) + # return main_menu_fallback() + + # if not fallback_needed: + # add_to_menu_history(key) + # if ret_f.__name__ != "ValidatePrefs": + # Dict["last_menu_item"] = key + # + add_to_menu_history(key) + Dict["last_menu_item"] = key + return ret_f(*ret_a, **ret_kw) + + # @route may be used multiple times + enforce_route = kwargs.pop("enforce_route", None) + if not already_wrapped: + inner.orig_f = f + + return enable_channel_wrapper(route(*args, **kwargs), enforce_route=enforce_route)(inner) + return enable_channel_wrapper(route(*args, **kwargs), enforce_route=enforce_route)(f) + + return wrap diff --git a/Contents/Code/interface/item_details.py b/Contents/Code/interface/item_details.py new file mode 100644 index 000000000..546ae9632 --- /dev/null +++ b/Contents/Code/interface/item_details.py @@ -0,0 +1,725 @@ +# coding=utf-8 +import os + +from collections import OrderedDict + +from subzero.language import Language + +from sub_mod import SubtitleModificationsMenu +from menu_helpers import debounce, SubFolderObjectContainer, default_thumb, add_incl_excl_options, get_item_task_data, \ + set_refresh_menu_state, route +from support.extract import extract_embedded_sub + +from refresh_item import RefreshItem +from subzero.constants import PREFIX +from support.config import config, TEXT_SUBTITLE_EXTS +from support.helpers import timestamp, df, get_language, display_language, get_language_from_stream +from support.items import get_item_kind_from_rating_key, get_item, get_current_sub, get_item_title, save_stored_sub +from support.plex_media import get_plex_metadata, get_part, get_embedded_subtitle_streams, is_stream_forced, \ + update_stream_info +from support.scanning import scan_videos +from support.scheduler import scheduler +from support.storage import get_subtitle_storage +from support.i18n import _ + + +# fixme: needs kwargs cleanup + +@route(PREFIX + '/item/{rating_key}/actions') +def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, randomize=None, header=None, + message=None): + """ + displays the item details menu of an item that doesn't contain any deeper tree, such as a movie or an episode + :param rating_key: + :param title: + :param base_title: + :param item_title: + :param randomize: + :return: + """ + from interface.main import InclExclMenu + + title = unicode(base_title) + " > " + unicode(title) if base_title else unicode(title) + item = plex_item = get_item(rating_key) + current_kind = get_item_kind_from_rating_key(rating_key) + + timeout = 30 + + oc = SubFolderObjectContainer( + title2=title, + replace_parent=True, + header=header, + message=message) + + if not item: + oc.add(DirectoryObject( + key=Callback( + ItemDetailsMenu, + rating_key=rating_key, + title=title, + base_title=base_title, + item_title=item_title, + randomize=timestamp()), + title=_(u"Item not found: %s!", item_title), + summary=_("Plex didn't return any information about the item, please refresh it and come back later"), + thumb=default_thumb + )) + return oc + + # add back to season for episode + if current_kind == "episode": + from interface.menu import MetadataMenu + show = get_item(item.show.rating_key) + season = get_item(item.season.rating_key) + + oc.add(DirectoryObject( + key=Callback( + MetadataMenu, + rating_key=season.rating_key, + title=season.title, + base_title=show.title, + previous_item_type="show", + previous_rating_key=show.rating_key, + display_items=True, + randomize=timestamp()), + title=_(u"< Back to %s", season.title), + summary=_("Back to %s > %s", show.title, season.title), + thumb=season.thumb or default_thumb + )) + + oc.add(DirectoryObject( + key=Callback( + RefreshItem, + rating_key=rating_key, + item_title=item_title, + randomize=timestamp(), + timeout=timeout * 1000), + title=_(u"Refresh: %s", item_title), + summary=_("Refreshes %(the_movie_series_season_episode)s, possibly searching for missing and picking up " + "new subtitles on disk", the_movie_series_season_episode=_(u"the %s" % current_kind)), + thumb=item.thumb or default_thumb + )) + oc.add(DirectoryObject( + key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp(), + timeout=timeout * 1000), + title=_(u"Force-find subtitles: %(item_title)s", item_title=item_title), + summary=_("Issues a forced refresh, ignoring known subtitles and searching for new ones"), + thumb=item.thumb or default_thumb + )) + + # get stored subtitle info for item id + subtitle_storage = get_subtitle_storage() + stored_subs = subtitle_storage.load_or_new(item) + + # look for subtitles for all available media parts and all of their languages + has_multiple_parts = len(plex_item.media) > 1 + part_index = 0 + for media in plex_item.media: + for part in media.parts: + filename = os.path.basename(part.file) + if not os.path.exists(part.file): + continue + + update_stream_info(part) + + part_id = str(part.id) + part_index += 1 + + part_index_addon = u"" + part_summary_addon = u"" + if has_multiple_parts: + part_index_addon = _(u"File %(file_part_index)s: ", file_part_index=part_index) + part_summary_addon = u"%s " % filename + + # iterate through all configured languages + for lang in config.lang_list: + # get corresponding stored subtitle data for that media part (physical media item), for language + current_sub = stored_subs.get_any(part_id, lang) + current_sub_id = None + current_sub_provider_name = None + + summary = _(u"%(part_summary)sNo current subtitle in storage", part_summary=part_summary_addon) + current_score = None + if current_sub: + current_sub_id = current_sub.id + current_sub_provider_name = current_sub.provider_name + current_score = current_sub.score + + summary = _(u"%(part_summary)sCurrent subtitle: %(provider_name)s (added: %(date_added)s, " + u"%(mode)s), Language: %(language)s, Score: %(score)i, Storage: %(storage_type)s", + part_summary=part_summary_addon, + provider_name=_(current_sub.provider_name), + date_added=df(current_sub.date_added), + mode=_(current_sub.mode_verbose), + language=display_language(lang), + score=current_sub.score, + storage_type=current_sub.storage_type) + + oc.add(DirectoryObject( + key=Callback(SubtitleOptionsMenu, rating_key=rating_key, part_id=part_id, title=title, + item_title=item_title, language=lang, language_name=display_language(lang), + current_id=current_sub_id, + item_type=plex_item.type, filename=filename, current_data=summary, + randomize=timestamp(), current_provider=current_sub_provider_name, + current_score=current_score), + title=_(u"%(part_summary)sManage %(language)s subtitle", part_summary=part_index_addon, + language=display_language(lang)), + summary=summary + )) + else: + oc.add(DirectoryObject( + key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, part_id=part_id, title=title, + item_title=item_title, language=lang, language_name=display_language(lang), + current_id=current_sub_id, + item_type=plex_item.type, filename=filename, current_data=summary, + randomize=timestamp(), current_provider=current_sub_provider_name, + current_score=current_score), + title=_(u"%(part_summary)sList %(language)s subtitles", part_summary=part_index_addon, + language=display_language(lang)), + summary=summary + )) + + if config.plex_transcoder: + # embedded subtitles + embedded_count = 0 + embedded_langs = [] + for stream in part.streams: + # subtitle stream + if stream.stream_type == 3 and not stream.stream_key and stream.codec in TEXT_SUBTITLE_EXTS: + lang = get_language_from_stream(stream.language_code) + is_forced = is_stream_forced(stream) + + if not lang and config.treat_und_as_first: + lang = list(config.lang_list)[0] + + if lang: + lang = Language.rebuild(lang, forced=is_forced) + embedded_langs.append(lang) + embedded_count += 1 + + if embedded_count: + oc.add(DirectoryObject( + key=Callback(ListEmbeddedSubsForItemMenu, rating_key=rating_key, part_id=part_id, title=title, + item_type=plex_item.type, item_title=item_title, base_title=base_title, + randomize=timestamp()), + title=_(u"%(part_summary)sEmbedded subtitles (%(languages)s)", + part_summary=part_index_addon, + languages=", ".join(display_language(l) + for l in list(OrderedDict.fromkeys(embedded_langs)))), + summary=_(u"Extract embedded subtitle streams") + )) + + ignore_title = item_title + if current_kind == "episode": + ignore_title = get_item_title(item) + add_incl_excl_options(oc, "videos", title=ignore_title, rating_key=rating_key, callback_menu=InclExclMenu) + subtitle_storage.destroy() + + return oc + + +@route(PREFIX + '/item/current_sub/{rating_key}/{part_id}') +def SubtitleOptionsMenu(**kwargs): + oc = SubFolderObjectContainer(title2=unicode(kwargs["title"]), replace_parent=True, header=kwargs.get("header"), + message=kwargs.get("message")) + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + language = kwargs["language"] + current_data = unicode(kwargs["current_data"]) + + current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language) + subs_count = stored_subs.count(part_id, language) + kwargs.pop("randomize") + + oc.add(DirectoryObject( + key=Callback(ItemDetailsMenu, rating_key=kwargs["rating_key"], item_title=kwargs["item_title"], + title=kwargs["title"], randomize=timestamp()), + title=_(u"< Back to %s", kwargs["title"]), + summary=current_data, + thumb=default_thumb + )) + if subs_count: + oc.add(DirectoryObject( + key=Callback(ListStoredSubsForItemMenu, randomize=timestamp(), **kwargs), + title=_(u"Select active %(language)s subtitle", language=kwargs["language_name"]), + summary=_(u"%(count)d subtitles in storage", count=subs_count) + )) + + oc.add(DirectoryObject( + key=Callback(ListAvailableSubsForItemMenu, randomize=timestamp(), **kwargs), + title=_(u"List available %(language)s subtitles", language=kwargs["language_name"]), + summary=current_data + )) + if current_sub: + oc.add(DirectoryObject( + key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs), + title=_(u"Modify current %(language)s subtitle", language=kwargs["language_name"]), + summary=_(u"Currently applied mods: %(mod_list)s", + mod_list=(", ".join(current_sub.mods) if current_sub.mods else "none")) + )) + + if current_sub.provider_name != "embedded": + oc.add(DirectoryObject( + key=Callback(BlacklistSubtitleMenu, randomize=timestamp(), **kwargs), + title=_(u"Blacklist current %(language)s subtitle and search for a new one", + language=kwargs["language_name"]), + summary=current_data + )) + + current_bl, subs = stored_subs.get_blacklist(part_id, language) + if current_bl: + oc.add(DirectoryObject( + key=Callback(ManageBlacklistMenu, randomize=timestamp(), **kwargs), + title=_(u"Manage blacklist (%(amount)s contained)", amount=len(current_bl)), + summary=_(u"Inspect currently blacklisted subtitles") + )) + + storage.destroy() + return oc + + +@route(PREFIX + '/item/list_stored_subs/{rating_key}/{part_id}') +def ListStoredSubsForItemMenu(**kwargs): + oc = SubFolderObjectContainer(title2=unicode(kwargs["title"]), replace_parent=True) + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + language = Language.fromietf(kwargs["language"]) + + current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language) + all_subs = stored_subs.get_all(part_id, language) + kwargs.pop("randomize") + + for key, subtitle in sorted(filter(lambda x: x[0] not in ("current", "blacklist"), all_subs.items()), + key=lambda x: x[1].date_added, reverse=True): + is_current = key == all_subs["current"] + + summary = _(u"added: %(date_added)s, %(mode)s, Language: %(language)s, Score: %(score)i, Storage: " + u"%(storage_type)s", + date_added=df(subtitle.date_added), + mode=_(subtitle.mode_verbose), + language=display_language(language), + score=subtitle.score, + storage_type=subtitle.storage_type) + + sub_name = subtitle.provider_name + if sub_name == "embedded": + sub_name += " (%s)" % subtitle.id + + oc.add(DirectoryObject( + key=Callback(SelectStoredSubForItemMenu, randomize=timestamp(), sub_key="__".join(key), **kwargs), + title=_(u"%(current_state)s%(subtitle_name)s, Score: %(score)s", + current_state=_("Current: ") if is_current else _("Stored: "), + subtitle_name=sub_name, + score=subtitle.score), + summary=summary + )) + + return oc + + +@route(PREFIX + '/item/set_current_sub/{rating_key}/{part_id}') +@debounce +def SelectStoredSubForItemMenu(**kwargs): + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + language = Language.fromietf(kwargs["language"]) + item_type = kwargs["item_type"] + sub_key = tuple(kwargs.pop("sub_key").split("__")) + + plex_item = get_item(rating_key) + storage = get_subtitle_storage() + stored_subs = storage.load(plex_item.rating_key) + + subtitles = stored_subs.get_all(part_id, language) + subtitle = subtitles[sub_key] + + save_stored_sub(subtitle, rating_key, part_id, language, item_type, plex_item=plex_item, storage=storage, + stored_subs=stored_subs) + + stored_subs.set_current(part_id, language, sub_key) + storage.save(stored_subs) + storage.destroy() + + kwa = { + "header": _("Success"), + "message": _("Subtitle saved to disk"), + "title": kwargs["title"], + "item_title": kwargs["item_title"], + "base_title": kwargs.get("base_title") + } + + # fixme: return to SubtitleOptionsMenu properly? (needs recomputation of current_data + + return ItemDetailsMenu(rating_key, randomize=timestamp(), **kwa) + + +@route(PREFIX + '/item/blacklist_recent/{language}') +@route(PREFIX + '/item/blacklist_recent') +def BlacklistRecentSubtitleMenu(**kwargs): + if "last_played_items" not in Dict or not Dict["last_played_items"]: + return + + rating_key = Dict["last_played_items"][0] + kwargs["rating_key"] = rating_key + return BlacklistAllPartsSubtitleMenu(**kwargs) + + +@route(PREFIX + '/item/blacklist_all/{rating_key}/{language}') +@route(PREFIX + '/item/blacklist_all/{rating_key}') +def BlacklistAllPartsSubtitleMenu(**kwargs): + rating_key = kwargs.get("rating_key") + language = kwargs.get("language") + if language: + language = Language.fromietf(language) + + item = get_item(rating_key) + + if not item: + return + + item_title = get_item_title(item) + + subtitle_storage = get_subtitle_storage() + stored_subs = subtitle_storage.load_or_new(item) + for part_id, languages in stored_subs.parts.iteritems(): + sub_dict = languages + if language: + key = str(language) + if key not in sub_dict: + continue + + sub_dict = {key: sub_dict[key]} + + for language, subs in sub_dict.iteritems(): + if "current" in subs: + stored_subs.blacklist(part_id, language, subs["current"]) + Log.Info("Added %s to blacklist", subs["current"]) + + subtitle_storage.save(stored_subs) + subtitle_storage.destroy() + + return RefreshItem(rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp(), timeout=30000) + + +def blacklist(rating_key, part_id, language): + current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language) + if not current_sub: + return + + stored_subs.blacklist(part_id, language, current_sub.key) + storage.save(stored_subs) + storage.destroy() + + Log.Info("Added %s to blacklist", current_sub.key) + + return True + + +@route(PREFIX + '/item/blacklist/{rating_key}/{part_id}') +@debounce +def BlacklistSubtitleMenu(**kwargs): + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + language = kwargs["language"] + item_title = kwargs["item_title"] + + blacklist(rating_key, part_id, language) + kwargs.pop("randomize") + + return RefreshItem(rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp(), timeout=30000) + + +@route(PREFIX + '/item/manage_blacklist/{rating_key}/{part_id}', force=bool) +@debounce +def ManageBlacklistMenu(**kwargs): + oc = SubFolderObjectContainer(title2=unicode(kwargs["title"]), replace_parent=True) + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + language = kwargs["language"] + remove_sub_key = kwargs.pop("remove_sub_key", None) + current_data = unicode(kwargs["current_data"]) + + current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language) + current_bl, subs = stored_subs.get_blacklist(part_id, language) + + if remove_sub_key: + remove_sub_key = tuple(remove_sub_key.split("__")) + stored_subs.blacklist(part_id, language, remove_sub_key, add=False) + storage.save(stored_subs) + Log.Info("Removed %s from blacklist", remove_sub_key) + + kwargs.pop("randomize") + + oc.add(DirectoryObject( + key=Callback(ItemDetailsMenu, rating_key=kwargs["rating_key"], item_title=kwargs["item_title"], + title=kwargs["title"], randomize=timestamp()), + title=_(u"< Back to %s", kwargs["title"]), + summary=current_data, + thumb=default_thumb + )) + + def sorter(pair): + # thanks RestrictedModule parser for messing with lambda (x, y) + return pair[1]["date_added"] + + for sub_key, data in sorted(current_bl.iteritems(), key=sorter, reverse=True): + provider_name, subtitle_id = sub_key + title = _(u"%(provider_name)s, %(subtitle_id)s (added: %(date_added)s, %(mode)s), Language: %(language)s, " + u"Score: %(score)i, Storage: %(storage_type)s", + provider_name=_(provider_name), + subtitle_id=subtitle_id, + date_added=df(data["date_added"]), + mode=_(current_sub.get_mode_verbose(data["mode"])), + language=display_language(Language.fromietf(language)), + score=data["score"], + storage_type=data["storage_type"]) + oc.add(DirectoryObject( + key=Callback(ManageBlacklistMenu, remove_sub_key="__".join(sub_key), randomize=timestamp(), **kwargs), + title=title, + summary=_(u"Remove subtitle from blacklist") + )) + + storage.destroy() + + return oc + + +@route(PREFIX + '/item/search/{rating_key}/{part_id}', force=bool) +def ListAvailableSubsForItemMenu(rating_key=None, part_id=None, title=None, item_title=None, filename=None, + item_type="episode", language=None, language_name=None, force=False, current_id=None, + current_data=None, + current_provider=None, current_score=None, randomize=None): + assert rating_key, part_id + + running = scheduler.is_task_running("AvailableSubsForItem") + search_results = get_item_task_data("AvailableSubsForItem", rating_key, language) + + current_data = unicode(current_data) if current_data else None + + if (search_results is None or force) and not running: + scheduler.dispatch_task("AvailableSubsForItem", rating_key=rating_key, item_type=item_type, part_id=part_id, + language=language) + running = True + + oc = SubFolderObjectContainer(title2=unicode(title), replace_parent=True) + oc.add(DirectoryObject( + key=Callback(ItemDetailsMenu, rating_key=rating_key, item_title=item_title, title=title, randomize=timestamp()), + title=_(u"< Back to %s", title), + summary=current_data, + thumb=default_thumb + )) + + metadata = get_plex_metadata(rating_key, part_id, item_type) + plex_part = None + if not config.low_impact_mode: + scanned_parts = scan_videos([metadata], ignore_all=True) + + if not scanned_parts: + Log.Error("Couldn't list available subtitles for %s", rating_key) + return oc + + video, plex_part = scanned_parts.items()[0] + + video_display_data = [video.format] if video.format else [] + if video.release_group: + video_display_data.append(unicode(_(u"by %(release_group)s", release_group=video.release_group))) + video_display_data = " ".join(video_display_data) + else: + video_display_data = metadata["filename"] + + current_display = (_(u"Current: %(provider_name)s (%(score)s) ", + provider_name=_(current_provider), + score=current_score if current_provider else "")) + if not running: + oc.add(DirectoryObject( + key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title, language=language, + filename=filename, part_id=part_id, title=title, current_id=current_id, force=True, + current_provider=current_provider, current_score=current_score, + current_data=current_data, item_type=item_type, randomize=timestamp()), + title=_(u"Search for %(language)s subs (%(video_data)s)", + language=get_language(language).name, + video_data=video_display_data), + summary=_(u"%(current_info)sFilename: %(filename)s", current_info=current_display, filename=filename), + thumb=default_thumb + )) + + if search_results == "found_none": + oc.add(DirectoryObject( + key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title, + language=language, filename=filename, current_data=current_data, force=True, + part_id=part_id, title=title, current_id=current_id, item_type=item_type, + current_provider=current_provider, current_score=current_score, + randomize=timestamp()), + title=_(u"No subtitles found"), + summary=_(u"%(current_info)sFilename: %(filename)s", current_info=current_display, filename=filename), + thumb=default_thumb + )) + else: + oc.add(DirectoryObject( + key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title, + language=language, filename=filename, current_data=current_data, + part_id=part_id, title=title, current_id=current_id, item_type=item_type, + current_provider=current_provider, current_score=current_score, + randomize=timestamp()), + title=_(u"Searching for %(language)s subs (%(video_data)s), refresh here ...", + language=display_language(get_language(language)), + video_data=video_display_data), + summary=_(u"%(current_info)sFilename: %(filename)s", current_info=current_display, filename=filename), + thumb=default_thumb + )) + + if not search_results or search_results == "found_none": + return oc + + current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language) + current_bl, subs = stored_subs.get_blacklist(part_id, language) + + seen = [] + for subtitle in search_results: + if subtitle.id in seen: + continue + + bl_addon = "" + if (str(subtitle.provider_name), str(subtitle.id)) in current_bl: + bl_addon = "Blacklisted " + + wrong_fps_addon = "" + wrong_series_addon = "" + wrong_season_ep_addon = "" + if subtitle.wrong_fps: + if plex_part: + wrong_fps_addon = _(" (wrong FPS, sub: %(subtitle_fps)s, media: %(media_fps)s)", + subtitle_fps=subtitle.fps, + media_fps=plex_part.fps) + else: + wrong_fps_addon = _(" (wrong FPS, sub: %(subtitle_fps)s, media: unknown, low impact mode)", + subtitle_fps=subtitle.fps) + + if subtitle.wrong_series: + wrong_series_addon = _(" (possibly wrong series)") + + if subtitle.wrong_season_ep: + wrong_season_ep_addon = _(" (possibly wrong season/episode)") + + oc.add(DirectoryObject( + key=Callback(TriggerDownloadSubtitle, rating_key=rating_key, randomize=timestamp(), item_title=item_title, + subtitle_id=str(subtitle.id), language=language), + title=_(u"%(blacklisted_state)s%(current_state)s: %(provider_name)s, score: %(score)s%(wrong_fps_state)s" + u"%(wrong_series_state)s%(wrong_season_ep_state)s", + blacklisted_state=bl_addon, + current_state=_("Available") if current_id != subtitle.id else _("Current"), + provider_name=_(subtitle.provider_name), + score=subtitle.score, + wrong_fps_state=wrong_fps_addon, + wrong_series_state=wrong_series_addon, + wrong_season_ep_state=wrong_season_ep_addon), + summary=_(u"Release: %(release_info)s, Matches: %(matches)s", + release_info=subtitle.release_info, + matches=", ".join(subtitle.matches)), + thumb=default_thumb + )) + + seen.append(subtitle.id) + + return oc + + +@route(PREFIX + '/download_subtitle/{rating_key}') +@debounce +def TriggerDownloadSubtitle(rating_key=None, subtitle_id=None, item_title=None, language=None, randomize=None): + from interface.main import fatality + + set_refresh_menu_state(_("Downloading subtitle for %(title_or_id)s", title_or_id=item_title or rating_key)) + search_results = get_item_task_data("AvailableSubsForItem", rating_key, language) + + download_subtitle = None + for subtitle in search_results: + if str(subtitle.id) == subtitle_id: + download_subtitle = subtitle + break + if not download_subtitle: + Log.Error(u"Something went horribly wrong") + + else: + scheduler.dispatch_task("DownloadSubtitleForItem", rating_key=rating_key, subtitle=download_subtitle) + + scheduler.clear_task_data("AvailableSubsForItem") + + return fatality(randomize=timestamp(), header=" ", replace_parent=True) + + +@route(PREFIX + '/item/embedded/{rating_key}/{part_id}') +def ListEmbeddedSubsForItemMenu(**kwargs): + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + title = kwargs["title"] + kwargs.pop("randomize") + + oc = SubFolderObjectContainer(title2=title, replace_parent=True) + + oc.add(DirectoryObject( + key=Callback(ItemDetailsMenu, rating_key=kwargs["rating_key"], item_title=kwargs["item_title"], + base_title=kwargs["base_title"], title=kwargs["item_title"], randomize=timestamp()), + title=_("< Back to %s", kwargs["title"]), + thumb=default_thumb + )) + + plex_item = get_item(rating_key) + part = get_part(plex_item, part_id) + + if part: + for stream_data in get_embedded_subtitle_streams(part, skip_duplicate_unknown=False): + language = stream_data["language"] + is_unknown = stream_data["is_unknown"] + stream = stream_data["stream"] + is_forced = stream_data["is_forced"] + + oc.add(DirectoryObject( + key=Callback(TriggerExtractEmbeddedSubForItemMenu, randomize=timestamp(), + stream_index=str(stream.index), language=language, with_mods=True, **kwargs), + title=_(u"Extract stream %(stream_index)s, %(language)s%(unknown_state)s%(forced_state)s" + u"%(stream_title)s with default mods", + stream_index=stream.index, + language=display_language(language), + unknown_state=_(" (unknown)") if is_unknown else "", + forced_state=_(" (forced)") if is_forced else "", + stream_title=" (\"%s\")" % stream.title if stream.title else ""), + )) + oc.add(DirectoryObject( + key=Callback(TriggerExtractEmbeddedSubForItemMenu, randomize=timestamp(), + stream_index=str(stream.index), language=language, **kwargs), + title=_(u"Extract stream %(stream_index)s, %(language)s%(unknown_state)s%(forced_state)s" + u"%(stream_title)s", + stream_index=stream.index, + language=display_language(language), + unknown_state=_(" (unknown)") if is_unknown else "", + forced_state=_(" (forced)") if is_forced else "", + stream_title=" (\"%s\")" % stream.title if stream.title else ""), + )) + return oc + + +@route(PREFIX + '/item/extract_embedded/{rating_key}/{part_id}/{stream_index}') +@debounce +def TriggerExtractEmbeddedSubForItemMenu(**kwargs): + rating_key = kwargs["rating_key"] + part_id = kwargs.get("part_id") + stream_index = kwargs.get("stream_index") + + Thread.Create(extract_embedded_sub, extract_mode="m", **kwargs) + header = _(u"Extracting of embedded subtitle %s of part %s:%s triggered", + stream_index, rating_key, part_id) + + kwargs.pop("randomize") + kwargs.pop("item_type") + kwargs.pop("stream_index") + kwargs.pop("part_id") + kwargs.pop("with_mods", False) + kwargs.pop("language") + kwargs["title"] = kwargs["item_title"] + kwargs["header"] = header + kwargs["message"] = header + + return ItemDetailsMenu(randomize=timestamp(), **kwargs) + + diff --git a/Contents/Code/interface/main.py b/Contents/Code/interface/main.py new file mode 100644 index 000000000..88eb49bbf --- /dev/null +++ b/Contents/Code/interface/main.py @@ -0,0 +1,472 @@ +# coding=utf-8 + +from subzero.constants import PREFIX, TITLE, ART +from support.config import config +from support.helpers import pad_title, timestamp, df, display_language +from support.scheduler import scheduler +from support.ignore import get_decision_list +from support.items import get_item_thumb, get_on_deck_items, get_all_items, get_items_info, get_item, get_item_title +from menu_helpers import main_icon, debounce, SubFolderObjectContainer, default_thumb, dig_tree, add_incl_excl_options, \ + ObjectContainer, route, handler +from support.i18n import _ +from item_details import ItemDetailsMenu + + +@handler(PREFIX, TITLE if not config.is_development else TITLE + " DEV", art=ART, thumb=main_icon) +@route(PREFIX) +def fatality(randomize=None, force_title=None, header=None, message=None, only_refresh=False, no_history=False, + replace_parent=False): + """ + subzero main menu + """ + from interface.advanced import PinMenu, ClearPin, AdvancedMenu + from interface.menu import RefreshMissing, IgnoreListMenu, HistoryMenu + + title = config.full_version # force_title if force_title is not None else config.full_version + oc = ObjectContainer(title1=title, title2=title, header=unicode(header) if header else title, message=message, + no_history=no_history, + replace_parent=replace_parent, no_cache=True) + + # always re-check permissions + config.refresh_permissions_status() + + # always re-check enabled sections + config.refresh_enabled_sections() + + if config.lock_menu and not config.pin_correct: + oc.add(DirectoryObject( + key=Callback(PinMenu, randomize=timestamp()), + title=pad_title(_("Enter PIN")), + summary=_("The owner has restricted the access to this menu. Please enter the correct pin"), + )) + return oc + + if not config.permissions_ok and config.missing_permissions: + if not isinstance(config.missing_permissions, list): + oc.add(DirectoryObject( + key=Callback(fatality, randomize=timestamp()), + title=pad_title(_("Insufficient permissions")), + summary=config.missing_permissions, + )) + else: + for title, path in config.missing_permissions: + oc.add(DirectoryObject( + key=Callback(fatality, randomize=timestamp()), + title=pad_title(_("Insufficient permissions")), + summary=_("Insufficient permissions on library %(title)s, folder: %(path)s", + title=title, + path=path), + )) + return oc + + if not config.enabled_sections: + oc.add(DirectoryObject( + key=Callback(fatality, randomize=timestamp()), + title=pad_title(_("I'm not enabled!")), + summary=_("Please enable me for some of your libraries in your server settings; currently I do nothing"), + )) + return oc + + if not only_refresh: + if Dict["current_refresh_state"]: + oc.add(DirectoryObject( + key=Callback(fatality, force_title=" ", randomize=timestamp()), + title=pad_title(_("Working ... refresh here")), + summary=_("Current state: %s; Last state: %s", + (Dict["current_refresh_state"] or _("Idle")) if "current_refresh_state" in Dict else _("Idle"), + (Dict["last_refresh_state"] or _("None")) if "last_refresh_state" in Dict else _("None") + ) + )) + + oc.add(DirectoryObject( + key=Callback(OnDeckMenu), + title=_("On-deck items"), + summary=_("Shows the current on deck items and allows you to individually (force-) refresh their metadata/subtitles."), + thumb=R("icon-ondeck.jpg") + )) + if "last_played_items" in Dict and Dict["last_played_items"]: + oc.add(DirectoryObject( + key=Callback(RecentlyPlayedMenu), + title=pad_title(_("Recently played items")), + summary=_("Shows the %s recently played items and allows you to individually (force-) refresh their metadata/subtitles.", config.store_recently_played_amount), + thumb=R("icon-played.jpg") + )) + oc.add(DirectoryObject( + key=Callback(RecentlyAddedMenu), + title=_("Recently-added items"), + summary=_("Shows the recently added items per section."), + thumb=R("icon-added.jpg") + )) + oc.add(DirectoryObject( + key=Callback(RecentMissingSubtitlesMenu, randomize=timestamp()), + title=_("Show recently added items with missing subtitles"), + summary=_("Lists items with missing subtitles. Click on \"Find recent items with missing subs\" to update list"), + thumb=R("icon-missing.jpg") + )) + oc.add(DirectoryObject( + key=Callback(SectionsMenu), + title=_("Browse all items"), + summary=_("Go through your whole library and manage your ignore list. You can also (force-) refresh the metadata/subtitles of individual items."), + thumb=R("icon-browse.jpg") + )) + + task_name = "SearchAllRecentlyAddedMissing" + task = scheduler.task(task_name) + + if task.ready_for_display: + task_state = _("Running: %(items_done)s/%(items_searching)s (%(percentage)s%%)", + items_done=task.items_done, + items_searching=task.items_searching, + percentage=task.percentage) + else: + lr = scheduler.last_run(task_name) + nr = scheduler.next_run(task_name) + task_state = _("Last run: %s; Next scheduled run: %s; Last runtime: %s", + df(scheduler.last_run(task_name)) if lr else "never", + df(scheduler.next_run(task_name)) if nr else "never", + str(task.last_run_time).split(".")[0]) + + oc.add(DirectoryObject( + key=Callback(RefreshMissing, randomize=timestamp()), + title=_("Search for missing subtitles (in recently-added items, max-age: %s)", Prefs[ + "scheduler.item_is_recent_age"]), + summary=_("Automatically run periodically by the scheduler, if configured. %s", task_state), + thumb=R("icon-search.jpg") + )) + + ref_list = get_decision_list() + incl_excl_ref = _("include list") if ref_list.store == "include" else _("ignore list") + + oc.add(DirectoryObject( + key=Callback(IgnoreListMenu), + title=_("Display %(incl_excl_list_name)s (%(count)d)", + incl_excl_list_name=incl_excl_ref, count=len(ref_list)), + summary=_("Show the current %(incl_excl_list_name)s (mainly used for the automatic tasks)", + incl_excl_list_name=incl_excl_ref), + thumb=R("icon-ignore.jpg") + )) + + oc.add(DirectoryObject( + key=Callback(HistoryMenu), + title=_("History"), + summary=_("Show the last %i downloaded subtitles", int(Prefs["history_size"])), + thumb=R("icon-history.jpg") + )) + + oc.add(DirectoryObject( + key=Callback(fatality, force_title=" ", randomize=timestamp()), + title=pad_title(_("Refresh")), + summary=_("Current state: %s; Last state: %s", + (Dict["current_refresh_state"] or _("Idle")) if "current_refresh_state" in Dict else _("Idle"), + (Dict["last_refresh_state"] or _("None")) if "last_refresh_state" in Dict else _("None") + ), + thumb=R("icon-refresh.jpg") + )) + + # add re-lock after pin unlock + if config.pin: + oc.add(DirectoryObject( + key=Callback(ClearPin, randomize=timestamp()), + title=pad_title(_("Re-lock menu(s)")), + summary=_("Enabled the PIN again for menu(s)") + )) + + if not only_refresh: + if "provider_throttle" in Dict and Dict["provider_throttle"].keys(): + summary_data = [] + for provider, data in Dict["provider_throttle"].iteritems(): + reason, until, desc = data + summary_data.append(unicode(_("%(throttled_provider)s until %(until_date)s (%(reason)s)", + throttled_provider=provider, + until_date=until.strftime("%y/%m/%d %H:%M"), + reason=reason))) + + oc.add(DirectoryObject( + key=Callback(fatality, force_title=" ", randomize=timestamp()), + title=pad_title(_("Throttled providers: %s", ", ".join(Dict["provider_throttle"].keys()))), + summary=", ".join(summary_data), + thumb=R("icon-throttled.jpg") + )) + + oc.add(DirectoryObject( + key=Callback(AdvancedMenu), + title=pad_title(_("Advanced functions")), + summary=_("Use at your own risk"), + thumb=R("icon-advanced.jpg") + )) + + return oc + + +@route(PREFIX + '/on_deck') +def OnDeckMenu(message=None): + """ + displays the items on deck + :param message: + :return: + """ + return mergedItemsMenu(title=_("Items On Deck"), base_title=_("Items On Deck"), itemGetter=get_on_deck_items) + + +@route(PREFIX + '/recently_played') +def RecentlyPlayedMenu(): + base_title = _("Recently Played") + oc = SubFolderObjectContainer(title2=base_title, replace_parent=True) + + for item in [get_item(rating_key) for rating_key in Dict["last_played_items"]]: + if not item: + continue + + if getattr(getattr(item, "__class__"), "__name__") not in ("Episode", "Movie"): + continue + + item_title = get_item_title(item) + + oc.add(DirectoryObject( + thumb=get_item_thumb(item) or default_thumb, + title=item_title, + key=Callback(ItemDetailsMenu, title=base_title + " > " + item.title, item_title=item.title, + rating_key=item.rating_key) + )) + + return oc + + +@route(PREFIX + '/recently_added') +def RecentlyAddedMenu(message=None): + """ + displays the items recently added per section + :param message: + :return: + """ + return SectionsMenu(base_title=_("Recently added"), section_items_key="recently_added", ignore_options=False) + + +@route(PREFIX + '/recent', force=bool) +@debounce +def RecentMissingSubtitlesMenu(force=False, randomize=None): + title = _("Items with missing subtitles") + oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True) + + running = scheduler.is_task_running("MissingSubtitles") + task_data = scheduler.get_task_data("MissingSubtitles") + missing_items = task_data["missing_subtitles"] if task_data else None + + if ((missing_items is None) or force) and not running: + scheduler.dispatch_task("MissingSubtitles") + running = True + + if not running: + oc.add(DirectoryObject( + key=Callback(RecentMissingSubtitlesMenu, force=True, randomize=timestamp()), + title=_(u"Find recent items with missing subtitles"), + thumb=default_thumb + )) + else: + oc.add(DirectoryObject( + key=Callback(RecentMissingSubtitlesMenu, force=False, randomize=timestamp()), + title=_(u"Updating, refresh here ..."), + thumb=default_thumb + )) + + if missing_items is not None: + for added_at, item_id, item_title, item, missing_languages in missing_items: + oc.add(DirectoryObject( + key=Callback(ItemDetailsMenu, title=title + " > " + item_title, item_title=item_title, + rating_key=item_id), + title=item_title, + summary=_("Missing: %s", ", ".join(display_language(l) for l in missing_languages)), + thumb=get_item_thumb(item) or default_thumb + )) + + return oc + + +def mergedItemsMenu(title, itemGetter, itemGetterKwArgs=None, base_title=None, *args, **kwargs): + """ + displays an item list of dynamic kinds of items + :param title: + :param itemGetter: + :param itemGetterKwArgs: + :param base_title: + :param args: + :param kwargs: + :return: + """ + oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True) + items = itemGetter(*args, **kwargs) + + for kind, title, item_id, deeper, item in items: + oc.add(DirectoryObject( + title=title, + key=Callback(ItemDetailsMenu, title=base_title + " > " + title, item_title=title, rating_key=item_id), + thumb=get_item_thumb(item) or default_thumb + )) + + return oc + + +def determine_section_display(kind, item, pass_kwargs=None): + """ + returns the menu function for a section based on the size of it (amount of items) + :param kind: + :param item: + :return: + """ + if pass_kwargs and pass_kwargs.get("section_items_key", "all") != "all": + return SectionMenu + if item.size > 80: + return SectionFirstLetterMenu + return SectionMenu + + +@route(PREFIX + '/incl_excl/set/{kind}/{rating_key}/{todo}/sure={sure}', kind=str, rating_key=str, todo=str, sure=bool) +def InclExclMenu(kind, rating_key, title=None, sure=False, todo="not_set"): + """ + displays the ignore options for a menu + :param kind: + :param rating_key: + :param title: + :param sure: + :param todo: + :return: + """ + ref_list = get_decision_list() + include = ref_list.store == "include" + list_str_ref = "include" if include else "ignore" + in_list = rating_key in ref_list[kind] + + if include: + # shortcut + sure = True + todo = "add" if not in_list else "remove" + + if not sure: + t = u"Add %(kind)s %(title)s to the ignore list" + if in_list: + t = u"Remove %(kind)s %(title)s from the ignore list" + oc = SubFolderObjectContainer(no_history=True, replace_parent=True, + title1=_(t, + kind=ref_list.verbose(kind), + title=title + ), + title2=_("Are you sure?")) + oc.add(DirectoryObject( + key=Callback(InclExclMenu, kind=kind, rating_key=rating_key, title=title, sure=True, + todo="add" if not in_list else "remove"), + title=pad_title(_("Are you sure?")), + )) + return oc + + rel = ref_list[kind] + dont_change = False + state = None + if todo == "remove": + if not in_list: + dont_change = True + else: + rel.remove(rating_key) + Log.Info("Removed %s (%s) from the %s list", title, rating_key, list_str_ref) + ref_list.remove_title(kind, rating_key) + ref_list.save() + elif todo == "add": + if in_list: + dont_change = True + else: + rel.append(rating_key) + Log.Info("Added %s (%s) to the %s list", title, rating_key, list_str_ref) + ref_list.add_title(kind, rating_key, title) + ref_list.save() + else: + dont_change = True + + if dont_change: + return fatality(force_title=" ", header=_("Didn't change the %(incl_excl_list_name)s", + incl_excl_list_name=_(list_str_ref)), no_history=True) + + if include: + t = "%(title)s added to the include list" + if todo == "remove": + t = "%(title)s removed from the include list" + else: + t = "%(title)s added to the ignore list" + if todo == "remove": + t = "%(title)s removed from the ignore list" + return fatality(force_title=" ", header=_(t, title=title,), no_history=True) + + +@route(PREFIX + '/sections') +def SectionsMenu(base_title=_("Sections"), section_items_key="all", ignore_options=True): + """ + displays the menu for all sections + :return: + """ + items = get_all_items("sections") + + return dig_tree(SubFolderObjectContainer(title2=_("Sections"), no_cache=True, no_history=True), items, None, + menu_determination_callback=determine_section_display, pass_kwargs={"base_title": base_title, + "section_items_key": section_items_key, + "ignore_options": ignore_options}, + fill_args={"title": "section_title"}) + + +@route(PREFIX + '/section', ignore_options=bool) +def SectionMenu(rating_key, title=None, base_title=None, section_title=None, ignore_options=True, + section_items_key="all"): + """ + displays the contents of a section + :param section_items_key: + :param rating_key: + :param title: + :param base_title: + :param section_title: + :param ignore_options: + :return: + """ + from menu import MetadataMenu + items = get_all_items(key=section_items_key, value=rating_key, base="library/sections") + + kind, deeper = get_items_info(items) + title = unicode(title) + + section_title = title + title = base_title + " > " + title + oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True) + if ignore_options: + add_incl_excl_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=InclExclMenu) + + return dig_tree(oc, items, MetadataMenu, + pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": "section", + "previous_rating_key": rating_key}) + + +@route(PREFIX + '/section/firstLetter', deeper=bool) +def SectionFirstLetterMenu(rating_key, title=None, base_title=None, section_title=None, ignore_options=True, + section_items_key="all"): + """ + displays the contents of a section indexed by its first char (A-Z, 0-9...) + :param ignore_options: ignored + :param section_items_key: ignored + :param rating_key: + :param title: + :param base_title: + :param section_title: + :return: + """ + from menu import FirstLetterMetadataMenu + items = get_all_items(key="first_character", value=rating_key, base="library/sections") + + kind, deeper = get_items_info(items) + + title = unicode(title) + oc = SubFolderObjectContainer(title2=section_title, no_cache=True, no_history=True) + title = base_title + " > " + title + add_incl_excl_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=InclExclMenu) + + oc.add(DirectoryObject( + key=Callback(SectionMenu, title=_("All"), base_title=title, rating_key=rating_key, ignore_options=False), + title="All" + ) + ) + return dig_tree(oc, items, FirstLetterMetadataMenu, force_rating_key=rating_key, fill_args={"key": "key"}, + pass_kwargs={"base_title": title, "display_items": deeper, "previous_rating_key": rating_key}) diff --git a/Contents/Code/interface/menu.py b/Contents/Code/interface/menu.py new file mode 100644 index 000000000..110bbbcc9 --- /dev/null +++ b/Contents/Code/interface/menu.py @@ -0,0 +1,401 @@ +# coding=utf-8 +import locale +import logging +import os +import platform +import traceback + +import logger +import copy + +from requests import HTTPError +from item_details import ItemDetailsMenu +from refresh_item import RefreshItem +from menu_helpers import add_incl_excl_options, dig_tree, set_refresh_menu_state, \ + default_thumb, debounce, ObjectContainer, SubFolderObjectContainer, route +from main import fatality, InclExclMenu +from advanced import DispatchRestart +from subzero.constants import ART, PREFIX, DEPENDENCY_MODULE_NAMES +from support.extract import season_extract_embedded +from support.scheduler import scheduler +from support.config import config +from support.helpers import timestamp, df, display_language +from support.ignore import get_decision_list +from support.items import get_all_items, get_items_info, get_item_kind_from_rating_key, get_item, get_item_title +from support.i18n import _ + +# init GUI +ObjectContainer.art = R(ART) +ObjectContainer.no_cache = True + +# default thumb for DirectoryObjects +DirectoryObject.thumb = default_thumb +Plugin.AddViewGroup("full_details", viewMode="InfoList", mediaType="items", type="list", summary=2) + + +@route(PREFIX + '/section/firstLetter/key', deeper=bool) +def FirstLetterMetadataMenu(rating_key, key, title=None, base_title=None, display_items=False, previous_item_type=None, + previous_rating_key=None): + """ + displays the contents of a section filtered by the first letter + :param rating_key: actually is the section's key + :param key: the firstLetter wanted + :param title: the first letter, or # + :param deeper: + :return: + """ + title = base_title + " > " + unicode(title) + oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True) + + items = get_all_items(key="first_character", value=[rating_key, key], base="library/sections", flat=False) + kind, deeper = get_items_info(items) + dig_tree(oc, items, MetadataMenu, + pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": kind, + "previous_rating_key": rating_key}) + return oc + + +@route(PREFIX + '/section/contents', display_items=bool) +def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, previous_item_type=None, + previous_rating_key=None, message=None, header=None, randomize=None): + """ + displays the contents of a section based on whether it has a deeper tree or not (movies->movie (item) list; series->series list) + :param rating_key: + :param title: + :param base_title: + :param display_items: + :param previous_item_type: + :param previous_rating_key: + :return: + """ + title = unicode(title) + item_title = title + title = base_title + " > " + title + oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True, header=header, message=message, + view_group="full_details") + + current_kind = get_item_kind_from_rating_key(rating_key) + + if display_items: + timeout = 30 + show = None + + # add back to series for season + if current_kind == "season": + timeout = 720 + + show = get_item(previous_rating_key) + oc.add(DirectoryObject( + key=Callback(MetadataMenu, rating_key=show.rating_key, title=show.title, base_title=show.section.title, + previous_item_type="section", display_items=True, randomize=timestamp()), + title=_(u"< Back to %s", show.title), + thumb=show.thumb or default_thumb + )) + elif current_kind == "series": + # it shouldn't take more than 6 minutes to scan all of a series' files and determine the force refresh + timeout = 3600 + + items = get_all_items(key="children", value=rating_key, base="library/metadata") + kind, deeper = get_items_info(items) + dig_tree(oc, items, MetadataMenu, + pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": kind, + "previous_rating_key": rating_key}) + + # we don't know exactly where we are here, only add ignore option to series + if current_kind in ("series", "season"): + item = get_item(rating_key) + sub_title = get_item_title(item) + add_incl_excl_options(oc, current_kind, title=sub_title, rating_key=rating_key, callback_menu=InclExclMenu) + + # mass-extract embedded + if config.plex_transcoder: + if current_kind == "season": + for lang in config.lang_list: + oc.add(DirectoryObject( + key=Callback(SeasonExtractEmbedded, rating_key=rating_key, language=lang, + base_title=show.section.title, display_items=display_items, item_title=item_title, + title=title, + previous_item_type=previous_item_type, with_mods=True, + previous_rating_key=previous_rating_key, randomize=timestamp()), + title=_(u"Extract missing %(language)s embedded subtitles", language=display_language(lang)), + summary=_("Extracts the not yet extracted embedded subtitles of all episodes for the current " + "season with all configured default modifications") + )) + oc.add(DirectoryObject( + key=Callback(SeasonExtractEmbedded, rating_key=rating_key, language=lang, + base_title=show.section.title, display_items=display_items, item_title=item_title, + title=title, force=True, + previous_item_type=previous_item_type, with_mods=True, + previous_rating_key=previous_rating_key, randomize=timestamp()), + title=_(u"Extract and activate %(language)s embedded subtitles", language=display_language(lang)), + summary=_("Extracts embedded subtitles of all episodes for the current season " + "with all configured default modifications") + )) + elif current_kind == "series": + for lang in config.lang_list: + oc.add(DirectoryObject( + key=Callback(SeasonExtractEmbedded, rating_key=rating_key, language=lang, + base_title=title, display_items=display_items, item_title=item_title, + title=title, mode="series", + previous_item_type=previous_item_type, with_mods=True, + previous_rating_key=previous_rating_key, randomize=timestamp()), + title=_(u"Extract missing %(language)s embedded subtitles", language=display_language(lang)), + summary=_("Extracts the not yet extracted embedded subtitles of all episodes for the current " + "series with all configured default modifications") + )) + oc.add(DirectoryObject( + key=Callback(SeasonExtractEmbedded, rating_key=rating_key, language=lang, + base_title=title, display_items=display_items, item_title=item_title, + title=title, force=True, mode="series", + previous_item_type=previous_item_type, with_mods=True, + previous_rating_key=previous_rating_key, randomize=timestamp()), + title=_(u"Extract and activate %(language)s embedded subtitles", language=display_language(lang)), + summary=_("Extracts embedded subtitles of all episodes for the current series " + "with all configured default modifications") + )) + + # add refresh + oc.add(DirectoryObject( + key=Callback(RefreshItem, rating_key=rating_key, item_title=title, refresh_kind=current_kind, + previous_rating_key=previous_rating_key, timeout=timeout * 1000, randomize=timestamp()), + title=_(u"Refresh: %s", item_title), + summary=_("Refreshes %(the_movie_series_season_episode)s, possibly searching for missing and picking up " + "new subtitles on disk", the_movie_series_season_episode=_(u"the %s" % current_kind)) + )) + oc.add(DirectoryObject( + key=Callback(RefreshItem, rating_key=rating_key, item_title=title, force=True, + refresh_kind=current_kind, previous_rating_key=previous_rating_key, timeout=timeout * 1000, + randomize=timestamp()), + title=_(u"Auto-Find subtitles: %s", item_title), + summary=_("Issues a forced refresh, ignoring known subtitles and searching for new ones") + )) + else: + return ItemDetailsMenu(rating_key=rating_key, title=title, item_title=item_title) + + return oc + + +@route(PREFIX + '/season/extract_embedded/{rating_key}/{language}') +def SeasonExtractEmbedded(**kwargs): + rating_key = kwargs.get("rating_key") + requested_language = kwargs.pop("language") + with_mods = kwargs.pop("with_mods") + item_title = kwargs.pop("item_title") + title = kwargs.pop("title") + force = kwargs.pop("force", False) + mode = kwargs.pop("mode", "season") + + Thread.Create(season_extract_embedded, **{"rating_key": rating_key, "requested_language": requested_language, + "with_mods": with_mods, "force": force, "mode": mode}) + + kwargs["header"] = _("Success") + kwargs["message"] = _(u"Extracting of embedded subtitles for %s triggered", title) + + kwargs.pop("randomize") + return MetadataMenu(randomize=timestamp(), title=item_title, **kwargs) + + +@route(PREFIX + '/ignore_list') +def IgnoreListMenu(): + ref_list = get_decision_list() + include = ref_list.store == "include" + list_title = _("Include list" if include else "Ignore list") + oc = SubFolderObjectContainer(title2=list_title, replace_parent=True) + for key in ref_list.key_order: + values = ref_list[key] + for value in values: + add_incl_excl_options(oc, key, title=ref_list.get_title(key, value), rating_key=value, + callback_menu=InclExclMenu) + return oc + + +@route(PREFIX + '/history') +def HistoryMenu(): + from support.history import get_history + history = get_history() + oc = SubFolderObjectContainer(title2=_("History"), replace_parent=True) + + for item in history.items[:100]: + possible_language = item.language + language_display = item.lang_name if not possible_language else display_language(possible_language) + + oc.add(DirectoryObject( + key=Callback(ItemDetailsMenu, title=item.title, item_title=item.item_title, + rating_key=item.rating_key), + title=u"%s (%s)" % (item.item_title, _(item.mode_verbose)), + summary=_(u"%s in %s (%s, score: %s), %s", language_display, item.section_title, + _(item.provider_name), item.score, df(item.time)), + thumb=item.thumb or default_thumb + )) + + history.destroy() + + return oc + + +@route(PREFIX + '/missing/refresh') +@debounce +def RefreshMissing(randomize=None): + scheduler.dispatch_task("SearchAllRecentlyAddedMissing") + header = "Refresh of recently added items with missing subtitles triggered" + return fatality(header=header, replace_parent=True) + + +def replace_item(obj, key, replace_value): + for k, v in obj.items(): + if isinstance(v, dict): + obj[k] = replace_item(v, key, replace_value) + if key in obj: + obj[key] = replace_value + return obj + + +def check_connections(): + # debug drone + Log.Debug("Checking connections ...") + log_buffer = [] + try: + from subliminal_patch.refiners.drone import SonarrClient, RadarrClient + log_buffer.append(["----- Connections -----"]) + for key, cls in [("sonarr", SonarrClient), ("radarr", RadarrClient)]: + if key in config.refiner_settings: + cname = key.capitalize() + try: + status = cls(**config.refiner_settings[key]).status(timeout=5) + except HTTPError, e: + if e.response.status_code == 401: + log_buffer.append(("%s: NOT WORKING - BAD API KEY", cname)) + else: + log_buffer.append(("%s: NOT WORKING - %s", cname, traceback.format_exc())) + except: + log_buffer.append(("%s: NOT WORKING - %s", cname, traceback.format_exc())) + else: + if status and status["version"]: + log_buffer.append(("%s: OK - %s", cname, status["version"])) + else: + log_buffer.append(("%s: NOT WORKING - %s", cname)) + except: + log_buffer.append(("Something went really wrong when evaluating Sonarr/Radarr: %s", traceback.format_exc())) + finally: + Core.log.setLevel(logging.DEBUG) + for entry in log_buffer: + Log.Debug(*entry) + + Core.log.setLevel(logging.getLevelName(Prefs["log_level"])) + + +@route(PREFIX + '/ValidatePrefs', enforce_route=True) +def ValidatePrefs(): + Core.log.setLevel(logging.DEBUG) + + if Prefs["log_console"]: + Core.log.addHandler(logger.console_handler) + Log.Debug("Logging to console from now on") + else: + Core.log.removeHandler(logger.console_handler) + Log.Debug("Stop logging to console") + + # cache the channel state + update_dict = False + restart = False + + # reset pin + Dict["pin_correct_time"] = None + + config.initialize() + if "channel_enabled" not in Dict: + update_dict = True + + elif Dict["channel_enabled"] != config.enable_channel: + Log.Debug("Interface features %s, restarting plugin", "enabled" if config.enable_channel else "disabled") + update_dict = True + restart = True + + if "plugin_pin_mode2" not in Dict: + update_dict = True + + elif Dict["plugin_pin_mode2"] != Prefs["plugin_pin_mode2"]: + update_dict = True + restart = True + + if update_dict: + Dict["channel_enabled"] = config.enable_channel + Dict["plugin_pin_mode2"] = Prefs["plugin_pin_mode2"] + Dict.Save() + + if restart: + scheduler.stop() + DispatchRestart() + return + + scheduler.setup_tasks() + scheduler.clear_task_data("MissingSubtitles") + set_refresh_menu_state(None) + + Log.Debug("Validate Prefs called.") + + # SZ config debug + Log.Debug("--- SZ Config-Debug ---") + for attr in [ + "version", "app_support_path", "data_path", "data_items_path", "enable_agent", + "enable_channel", "permissions_ok", "missing_permissions", "fs_encoding", + "subtitle_destination_folder", "include", "include_exclude_paths", "include_exclude_sz_files", + "new_style_cache", "dbm_supported", "lang_list", "providers", "normal_subs", "forced_only", "forced_also", + "plex_transcoder", "refiner_settings", "unrar", "adv_cfg_path", "use_custom_dns", + "has_anticaptcha", "anticaptcha_cls", "mediainfo_bin"]: + + value = getattr(config, attr) + if isinstance(value, dict): + d = replace_item(copy.deepcopy(value), "api_key", "xxxxxxxxxxxxxxxxxxxxxxxxx") + Log.Debug("config.%s: %s", attr, d) + continue + + if attr in ("api_key",): + value = "xxxxxxxxxxxxxxxxxxxxxxxxx" + + Log.Debug("config.%s: %s", attr, value) + + for attr in ["plugin_log_path", "server_log_path"]: + value = getattr(config, attr) + + if value: + access = os.access(value, os.R_OK) + if Core.runtime.os == "Windows": + try: + f = open(value, "r") + f.read(1) + f.close() + except: + access = False + + Log.Debug("config.%s: %s (accessible: %s)", attr, value, access) + + for attr in [ + "subtitles.save.filesystem", ]: + Log.Debug("Pref.%s: %s", attr, Prefs[attr]) + + if "sonarr" in config.refiner_settings or "radarr" in config.refiner_settings: + Thread.Create(check_connections) + + # fixme: check existance of and os access of logs + Log.Debug("----- Environment -----") + Log.Debug("Platform: %s", Core.runtime.platform) + Log.Debug("OS: %s", Core.runtime.os) + Log.Debug("Python: %s", platform.python_version()) + for key, value in os.environ.iteritems(): + if key.startswith("PLEX") or key.startswith("SZ_"): + if "TOKEN" in key: + outval = "xxxxxxxxxxxxxxxxxxx" + + else: + outval = value + Log.Debug("%s: %s", key, outval) + Log.Debug("Locale: %s", locale.getdefaultlocale()) + Log.Debug("-----------------------") + + Log.Debug("Setting log-level to %s", Prefs["log_level"]) + logger.register_logging_handler(DEPENDENCY_MODULE_NAMES, level=Prefs["log_level"]) + Core.log.setLevel(logging.getLevelName(Prefs["log_level"])) + os.environ['U1pfT01EQl9LRVk'] = '789CF30DAC2C8B0AF433F5C9AD34290A712DF30D7135F12D0FB3E502006FDE081E' + + return diff --git a/Contents/Code/interface/menu_helpers.py b/Contents/Code/interface/menu_helpers.py new file mode 100644 index 000000000..d39dd87b5 --- /dev/null +++ b/Contents/Code/interface/menu_helpers.py @@ -0,0 +1,198 @@ +# coding=utf-8 +import types +import datetime + +from func import enable_channel_wrapper, route_wrapper +from support.i18n import is_localized_string, _ +from support.items import get_item_thumb +from support.helpers import get_video_display_title, pad_title +from support.ignore import get_decision_list +from support.lib import get_intent +from support.config import config +from subzero.constants import ICON_SUB, ICON +from support.scheduler import scheduler + +default_thumb = R(ICON_SUB) +main_icon = ICON if not config.is_development else "icon-dev.jpg" + +# noinspection PyUnboundLocalVariable +route = route_wrapper +# noinspection PyUnboundLocalVariable +handler = enable_channel_wrapper(handler) + + +def add_incl_excl_options(oc, kind, callback_menu=None, title=None, rating_key=None, add_kind=True): + """ + + :param oc: oc to add our options to + :param kind: movie, show, episode ... - gets translated to the ignore key (sections, series, items) + :param callback_menu: menu to inject + :param title: + :param rating_key: + :return: + """ + # try to translate kind to the ignore key + use_kind = kind + ref_list = get_decision_list() + if kind not in ref_list: + use_kind = ref_list.translate_key(kind) + if not use_kind or use_kind not in ref_list: + return + + in_list = rating_key in ref_list[use_kind] + include = ref_list.store == "include" + + if include: + t = u"Enable Sub-Zero for %(kind)s \"%(title)s\"" + if in_list: + t = u"Disable Sub-Zero for %(kind)s \"%(title)s\"" + else: + t = u"Ignore %(kind)s \"%(title)s\"" + if in_list: + t = u"Un-ignore %(kind)s \"%(title)s\"" + + oc.add(DirectoryObject( + key=Callback(callback_menu, kind=use_kind, sure=False, todo="not_set", rating_key=str(rating_key), title=title), + title=_(t, + kind=ref_list.verbose(kind) if add_kind else "", + title=unicode(title)) + ) + ) + + +def dig_tree(oc, items, menu_callback, menu_determination_callback=None, force_rating_key=None, fill_args=None, + pass_kwargs=None, thumb=default_thumb): + for kind, title, key, dig_deeper, item in items: + thumb = get_item_thumb(item) or thumb + + add_kwargs = {} + if fill_args: + add_kwargs = dict((name, getattr(item, k)) for k, name in fill_args.iteritems() if item and hasattr(item, k)) + if pass_kwargs: + add_kwargs.update(pass_kwargs) + + # force details view for show/season + summary = " " if kind in ("show", "season") else None + + oc.add(DirectoryObject( + key=Callback(menu_callback or menu_determination_callback(kind, item, pass_kwargs=pass_kwargs), title=title, + rating_key=force_rating_key or key, **add_kwargs), + title=pad_title(title) if kind in ("show", "season") else title, thumb=thumb, summary=summary + )) + return oc + + +def set_refresh_menu_state(state_or_media, media_type="movies"): + """ + + :param state_or_media: string, None, or Media argument from Agent.update() + :param media_type: movies or series + :return: + """ + if not state_or_media: + # store it in last state and remove the current + Dict["last_refresh_state"] = Dict["current_refresh_state"] + Dict["current_refresh_state"] = None + Dict.Save() + return + + if isinstance(state_or_media, types.StringTypes) or is_localized_string(state_or_media): + Dict["current_refresh_state"] = unicode(state_or_media) + Dict.Save() + return + + media = state_or_media + media_id = media.id + title = None + if media_type == "series": + for season in media.seasons: + for episode in media.seasons[season].episodes: + ep = media.seasons[season].episodes[episode] + media_id = ep.id + title = get_video_display_title(_("show"), ep.title, parent_title=media.title, season=int(season), episode=int(episode)) + else: + title = get_video_display_title(_("movie"), media.title) + + intent = get_intent() + force_refresh = intent.get("force", media_id) + + t = u"Refreshing %(title)s" + if force_refresh: + t = u"Force-refreshing %(title)s" + + Dict["current_refresh_state"] = unicode(_(t, + title=unicode(title))) + Dict.Save() + + +def get_item_task_data(task_name, rating_key, language): + task_data = scheduler.get_task_data(task_name) + search_results = task_data.get(rating_key, {}) if task_data else {} + return search_results.get(language) + + +def debounce(func): + """ + prevent func from being called twice with the same arguments + :param func: + :return: + """ + + func.debounce = True + + return func + + +class SZObjectContainer(ObjectContainer): + def __init__(self, *args, **kwargs): + skip_pin_lock = kwargs.pop("skip_pin_lock", False) + + super(SZObjectContainer, self).__init__(*args, **kwargs) + + if (config.lock_menu or config.lock_advanced_menu) and not config.pin_correct and not skip_pin_lock: + config.locked = True + + def add(self, *args, **kwargs): + # disable self.add if we're in lockdown + container = args[0] + current_menu_target = container.key.split("?")[0] + is_pin_menu = current_menu_target.endswith("/pin") + + if config.locked and config.lock_menu and not is_pin_menu: + return + return super(SZObjectContainer, self).add(*args, **kwargs) + + +OriginalObjectContainer = ObjectContainer +ObjectContainer = SZObjectContainer + + +class SubFolderObjectContainer(ObjectContainer): + def __init__(self, *args, **kwargs): + super(SubFolderObjectContainer, self).__init__(*args, **kwargs) + from interface.menu import fatality + from support.helpers import pad_title, timestamp + self.add(DirectoryObject( + key=Callback(fatality, force_title=" ", randomize=timestamp()), + title=pad_title(_("<< Back to home")), + summary=_("Current state: %s; Last state: %s", + (Dict["current_refresh_state"] or _("Idle")) if "current_refresh_state" in Dict else _("Idle"), + (Dict["last_refresh_state"] or _("None")) if "last_refresh_state" in Dict else _("None") + ) + )) + + +ObjectClass = getattr(getattr(Redirect, "_object_class"), "__bases__")[0] + + +class ZipObject(ObjectClass): + def __init__(self, data): + ObjectClass.__init__(self, "") + self.zipdata = data + self.SetHeader("Content-Type", "application/zip") + + def Content(self): + self.SetHeader("Content-Disposition", + 'attachment; filename="' + datetime.datetime.now().strftime("Logs_%y%m%d_%H-%M-%S.zip") + + '"') + return self.zipdata diff --git a/Contents/Code/interface/refresh_item.py b/Contents/Code/interface/refresh_item.py new file mode 100644 index 000000000..d716f1afb --- /dev/null +++ b/Contents/Code/interface/refresh_item.py @@ -0,0 +1,32 @@ +# coding=utf-8 + +from subzero.constants import PREFIX +from menu_helpers import debounce, set_refresh_menu_state, route +from support.items import refresh_item +from support.helpers import timestamp +from support.i18n import _ + + +@route(PREFIX + '/item/refresh/{rating_key}/force', force=True) +@route(PREFIX + '/item/refresh/{rating_key}') +@debounce +def RefreshItem(rating_key=None, came_from="/recent", item_title=None, force=False, refresh_kind=None, + previous_rating_key=None, timeout=8000, randomize=None, trigger=True): + assert rating_key + from interface.main import fatality + header = " " + if trigger: + t = u"Triggering refresh for %(title)s" + if force: + u"Triggering forced refresh for %(title)s" + set_refresh_menu_state(_(t, + title=item_title)) + Thread.Create(refresh_item, rating_key=rating_key, force=force, refresh_kind=refresh_kind, + parent_rating_key=previous_rating_key, timeout=int(timeout)) + + t = u"Refresh of item %(item_id)s triggered" + if force: + t = u"Forced refresh of item %(item_id)s triggered" + header = _(t, + item_id=rating_key) + return fatality(randomize=timestamp(), header=header, replace_parent=True) diff --git a/Contents/Code/interface/sub_mod.py b/Contents/Code/interface/sub_mod.py new file mode 100644 index 000000000..6b62f5c03 --- /dev/null +++ b/Contents/Code/interface/sub_mod.py @@ -0,0 +1,284 @@ +# coding=utf-8 + +import traceback +import types + +from subzero.language import Language + +from menu_helpers import debounce, SubFolderObjectContainer, default_thumb, route +from subzero.modification import registry as mod_registry, SubtitleModifications +from subzero.constants import PREFIX +from support.plex_media import get_plex_metadata +from support.scanning import scan_videos +from support.helpers import timestamp, pad_title +from support.items import get_current_sub, set_mods_for_part +from support.i18n import _ + + +@route(PREFIX + '/item/sub_mods/{rating_key}/{part_id}', force=bool) +def SubtitleModificationsMenu(**kwargs): + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + language = kwargs["language"] + lang_instance = Language.fromietf(language) + current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language) + kwargs.pop("randomize") + + current_mods = current_sub.mods or [] + + oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True) + + from interface.item_details import SubtitleOptionsMenu + oc.add(DirectoryObject( + key=Callback(SubtitleOptionsMenu, randomize=timestamp(), **kwargs), + title=_(u"< Back to subtitle options for: %s", kwargs["title"]), + summary=unicode(kwargs["current_data"]), + thumb=default_thumb + )) + + for identifier, mod in mod_registry.mods.iteritems(): + if mod.advanced: + continue + + if mod.exclusive and identifier in current_mods: + continue + + if mod.languages and lang_instance not in mod.languages: + continue + + oc.add(DirectoryObject( + key=Callback(SubtitleSetMods, mods=identifier, mode="add", randomize=timestamp(), **kwargs), + title=pad_title(_(mod.description)), summary=_(mod.long_description) or "" + )) + + fps_mod = SubtitleModifications.get_mod_class("change_FPS") + oc.add(DirectoryObject( + key=Callback(SubtitleFPSModMenu, randomize=timestamp(), **kwargs), + title=pad_title(_(fps_mod.description)), summary=_(fps_mod.long_description) or "" + )) + + shift_mod = SubtitleModifications.get_mod_class("shift_offset") + oc.add(DirectoryObject( + key=Callback(SubtitleShiftModUnitMenu, randomize=timestamp(), **kwargs), + title=pad_title(_(shift_mod.description)), summary=_(shift_mod.long_description) or "" + )) + + color_mod = SubtitleModifications.get_mod_class("color") + oc.add(DirectoryObject( + key=Callback(SubtitleColorModMenu, randomize=timestamp(), **kwargs), + title=pad_title(_(color_mod.description)), summary=_(color_mod.long_description) or "" + )) + + if current_mods: + oc.add(DirectoryObject( + key=Callback(SubtitleSetMods, mods=None, mode="remove_last", randomize=timestamp(), **kwargs), + title=pad_title(_("Remove last applied mod (%s)", current_mods[-1])), + summary=_(u"Currently applied mods: %(mod_list)s", mod_list=", ".join(current_mods) if current_mods else _("none")) + )) + oc.add(DirectoryObject( + key=Callback(SubtitleListMods, randomize=timestamp(), **kwargs), + title=pad_title(_("Manage applied mods")), + summary=_(u"Currently applied mods: %(mod_list)s", mod_list=", ".join(current_mods)) + )) + oc.add(DirectoryObject( + key=Callback(SubtitleReapplyMods, randomize=timestamp(), **kwargs), + title=pad_title(_("Reapply applied mods")), + summary=_(u"Currently applied mods: %(mod_list)s", mod_list=", ".join(current_mods) if current_mods else _("none")) + )) + + oc.add(DirectoryObject( + key=Callback(SubtitleSetMods, mods=None, mode="clear", randomize=timestamp(), **kwargs), + title=pad_title(_("Restore original version")), + summary=_(u"Currently applied mods: %(mod_list)s", mod_list=", ".join(current_mods) if current_mods else _("none")) + )) + + storage.destroy() + + return oc + + +@route(PREFIX + '/item/sub_mod_fps/{rating_key}/{part_id}', force=bool) +def SubtitleFPSModMenu(**kwargs): + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + item_type = kwargs["item_type"] + + kwargs.pop("randomize") + + oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True) + + oc.add(DirectoryObject( + key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs), + title=_("< Back to subtitle modification menu") + )) + + metadata = get_plex_metadata(rating_key, part_id, item_type) + scanned_parts = scan_videos([metadata], ignore_all=True, skip_hashing=True) + video, plex_part = scanned_parts.items()[0] + + target_fps = plex_part.fps + + for fps in ["23.980", "23.976", "24.000", "25.000", "29.970", "30.000", "50.000", "59.940", "60.000"]: + if float(fps) == float(target_fps): + continue + + if float(fps) > float(target_fps): + indicator = _("subs constantly getting faster") + else: + indicator = _("subs constantly getting slower") + + mod_ident = SubtitleModifications.get_mod_signature("change_FPS", **{"from": fps, "to": target_fps}) + oc.add(DirectoryObject( + key=Callback(SubtitleSetMods, mods=mod_ident, mode="add", randomize=timestamp(), **kwargs), + title=_("%(from_fps)s fps -> %(to_fps)s fps (%(slower_or_faster_indicator)s)", + from_fps=fps, + to_fps=target_fps, + slower_or_faster_indicator=indicator) + )) + + return oc + + +POSSIBLE_UNITS = (("ms", "milliseconds"), ("s", "seconds"), ("m", "minutes"), ("h", "hours")) +POSSIBLE_UNITS_D = dict(POSSIBLE_UNITS) + + +@route(PREFIX + '/item/sub_mod_shift_unit/{rating_key}/{part_id}', force=bool) +def SubtitleShiftModUnitMenu(**kwargs): + oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True) + + kwargs.pop("randomize") + + oc.add(DirectoryObject( + key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs), + title=_("< Back to subtitle modifications") + )) + + for unit, title in POSSIBLE_UNITS: + oc.add(DirectoryObject( + key=Callback(SubtitleShiftModMenu, unit=unit, randomize=timestamp(), **kwargs), + title=_("Adjust by %(time_and_unit)s", time_and_unit=title) + )) + + return oc + + +@route(PREFIX + '/item/sub_mod_shift/{rating_key}/{part_id}/{unit}', force=bool) +def SubtitleShiftModMenu(unit=None, **kwargs): + if unit not in POSSIBLE_UNITS_D: + raise NotImplementedError + + kwargs.pop("randomize") + + oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True) + + oc.add(DirectoryObject( + key=Callback(SubtitleShiftModUnitMenu, randomize=timestamp(), **kwargs), + title=_("< Back to unit selection") + )) + + rng = [] + if unit == "h": + rng = list(reversed(range(-10, 0))) + list(reversed(range(1, 11))) + elif unit in ("m", "s"): + rng = list(reversed(range(-15, 0))) + list(reversed(range(1, 16))) + elif unit == "ms": + rng = list(reversed(range(-900, 0, 100))) + list(reversed(range(100, 1000, 100))) + + for i in rng: + if i == 0: + continue + + mod_ident = SubtitleModifications.get_mod_signature("shift_offset", **{unit: i}) + oc.add(DirectoryObject( + key=Callback(SubtitleSetMods, mods=mod_ident, mode="add", randomize=timestamp(), **kwargs), + title="%s %s" % (("%s" if i < 0 else "+%s") % i, unit) + )) + + return oc + + +@route(PREFIX + '/item/sub_mod_colors/{rating_key}/{part_id}', force=bool) +def SubtitleColorModMenu(**kwargs): + kwargs.pop("randomize") + + color_mod = SubtitleModifications.get_mod_class("color") + + oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True) + + oc.add(DirectoryObject( + key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs), + title=_("< Back to subtitle modification menu") + )) + + for color, code in color_mod.colors.iteritems(): + mod_ident = SubtitleModifications.get_mod_signature("color", **{"name": color}) + oc.add(DirectoryObject( + key=Callback(SubtitleSetMods, mods=mod_ident, mode="add", randomize=timestamp(), **kwargs), + title="%s (%s)" % (color, code) + )) + + return oc + + +@route(PREFIX + '/item/sub_set_mods/{rating_key}/{part_id}/{mods}/{mode}', force=bool) +@debounce +def SubtitleSetMods(mods=None, mode=None, **kwargs): + if not isinstance(mods, types.ListType) and mods: + mods = [mods] + + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + lang_a2 = kwargs["language"] + item_type = kwargs["item_type"] + + language = Language.fromietf(lang_a2) + + set_mods_for_part(rating_key, part_id, language, item_type, mods, mode=mode) + + kwargs.pop("randomize") + return SubtitleModificationsMenu(randomize=timestamp(), **kwargs) + + +@route(PREFIX + '/item/sub_reapply_mods/{rating_key}/{part_id}', force=bool) +@debounce +def SubtitleReapplyMods(**kwargs): + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + lang_a2 = kwargs["language"] + item_type = kwargs["item_type"] + + language = Language.fromietf(lang_a2) + + set_mods_for_part(rating_key, part_id, language, item_type, [], mode="add") + + kwargs.pop("randomize") + return SubtitleModificationsMenu(randomize=timestamp(), **kwargs) + + +@route(PREFIX + '/item/sub_list_mods/{rating_key}/{part_id}', force=bool) +@debounce +def SubtitleListMods(**kwargs): + rating_key = kwargs["rating_key"] + part_id = kwargs["part_id"] + language = kwargs["language"] + current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language) + + kwargs.pop("randomize") + + oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True) + + oc.add(DirectoryObject( + key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs), + title=_("< Back to subtitle modifications") + )) + + for identifier in current_sub.mods: + oc.add(DirectoryObject( + key=Callback(SubtitleSetMods, mods=identifier, mode="remove", randomize=timestamp(), **kwargs), + title=_("Remove: %(mod_name)s", mod_name=identifier) + )) + + storage.destroy() + + return oc \ No newline at end of file diff --git a/Contents/Code/logger.py b/Contents/Code/logger.py index 1a8a803e9..b670f1fc2 100644 --- a/Contents/Code/logger.py +++ b/Contents/Code/logger.py @@ -1,15 +1,22 @@ import logging -def registerLoggingHander(dependencies): - plexHandler = PlexLoggerHandler() - for dependency in dependencies: - Log.Debug("Registering LoggerHandler for dependency: %s" % dependency) + +def register_logging_handler(dependencies, level="ERROR"): + plex_handler = PlexLoggerHandler() + for dependency in dependencies: + Log.Debug("Registering LoggerHandler for dependency: %s" % dependency) log = logging.getLogger(dependency) - log.setLevel('DEBUG') - log.addHandler(plexHandler) + # remove previous plex logging handlers + # fixme: this is not the most elegant solution... + for handler in log.handlers: + if isinstance(handler, PlexLoggerHandler): + log.removeHandler(handler) + + log.setLevel(level) + log.addHandler(plex_handler) + class PlexLoggerHandler(logging.StreamHandler): - def __init__(self, level=0): super(PlexLoggerHandler, self).__init__(level) @@ -30,4 +37,9 @@ def emit(self, record): elif record.levelno == logging.FATAL: Log.Exception(self.getFormattedString(record)) else: - Log.Error("UNKNOWN LEVEL: %s", record.getMessage()) \ No newline at end of file + Log.Error("UNKNOWN LEVEL: %s", record.getMessage()) + + +console_handler = logging.StreamHandler() +console_formatter = Framework.core.LogFormatter('%(asctime)-15s - %(name)-32s (%(thread)x) : %(levelname)s (%(module)s:%(lineno)d) - %(message)s') +console_handler.setFormatter(console_formatter) diff --git a/Contents/Code/support/LICENSE b/Contents/Code/support/LICENSE new file mode 100644 index 000000000..6b3068f16 --- /dev/null +++ b/Contents/Code/support/LICENSE @@ -0,0 +1,6 @@ +License for parts taken out of plexinc-agents/LocalMedia.bundle + +License +------- + +If the software submitted to this repository accesses or calls any software provided by Plex (“Interfacing Software”), then as a condition for receiving services from Plex in response to such accesses or calls, you agree to grant and do hereby grant to Plex and its affiliates worldwide a worldwide, nonexclusive, and royalty-free right and license to use (including testing, hosting and linking to), copy, publicly perform, publicly display, reproduce in copies for distribution, and distribute the copies of any Interfacing Software made by you or with your assistance; provided, however, that you may notify Plex at legal@plex.tv if you do not wish for Plex to use, distribute, copy, publicly perform, publicly display, reproduce in copies for distribution, or distribute copies of an Interfacing Software that was created by you, and Plex will reasonable efforts to comply with such a request within a reasonable time. \ No newline at end of file diff --git a/Contents/Code/support/__init__.py b/Contents/Code/support/__init__.py new file mode 100644 index 000000000..e3dc2a581 --- /dev/null +++ b/Contents/Code/support/__init__.py @@ -0,0 +1,76 @@ +import sys +# thanks, https://github.com/trakt/Plex-Trakt-Scrobbler/blob/master/Trakttv.bundle/Contents/Code/core/__init__.py + +import config + +sys.modules["support.config"] = config + +import helpers + +sys.modules["support.helpers"] = helpers + +import lib + +sys.modules["support.lib"] = lib + +import i18n + +sys.modules["support.i18n"] = i18n + +helpers._ = i18n._ + +import history + +sys.modules["support.history"] = history + +import plex_media +sys.modules["support.plex_media"] = plex_media + +import localmedia + +sys.modules["support.localmedia"] = localmedia + +import subtitlehelpers + +sys.modules["support.subtitlehelpers"] = subtitlehelpers + +import items + +sys.modules["support.items"] = items + +import scheduler + +sys.modules["support.scheduler"] = scheduler + +import storage + +sys.modules["support.storage"] = storage + +import scanning +sys.modules["support.scanning"] = scanning + +import missing_subtitles + +sys.modules["support.missing_subtitles"] = missing_subtitles + +import extract + +sys.modules["support.extract"] = extract + +import tasks + +sys.modules["support.tasks"] = tasks + +import ignore + +sys.modules["support.ignore"] = ignore + +import data + +sys.modules["support.data"] = data + +import activities +sys.modules["support.activities"] = activities + +import download +sys.modules["support.download"] = download \ No newline at end of file diff --git a/Contents/Code/support/activities.py b/Contents/Code/support/activities.py new file mode 100644 index 000000000..0e4cf42a6 --- /dev/null +++ b/Contents/Code/support/activities.py @@ -0,0 +1,132 @@ +# coding=utf-8 +from wraptor.decorators import throttle +from config import config +from items import get_item, get_item_kind_from_item, refresh_item + +Activity = None +try: + from plex_activity import Activity +except ImportError: + pass + + +class PlexActivityManager(object): + def start(self): + activity_sources_enabled = None + + if not Activity: + return + + if config.plex_token: + from plex import Plex + Plex.configuration.defaults.authentication(config.plex_token) + activity_sources_enabled = ["websocket"] + Activity.on('websocket.playing', self.on_playing) + + if activity_sources_enabled: + Activity.start(activity_sources_enabled) + + @throttle(5, instance_method=True) + def on_playing(self, info): + # ignore non-playing states and anything too far in + if info["state"] != "playing" or info["viewOffset"] > 60000: + return + + # don't trigger on the first hit ever + if "last_played_items" not in Dict: + Dict["last_played_items"] = [] + Dict.Save() + return + + rating_key = info["ratingKey"] + + # only use integer based rating keys + try: + int(rating_key) + except ValueError: + return + + if rating_key in Dict["last_played_items"] and rating_key != Dict["last_played_items"][0]: + # shift last played + Dict["last_played_items"].insert(0, + Dict["last_played_items"].pop(Dict["last_played_items"].index(rating_key))) + Dict.Save() + + elif rating_key not in Dict["last_played_items"]: + # new playing; store last X recently played items + Dict["last_played_items"].insert(0, rating_key) + Dict["last_played_items"] = Dict["last_played_items"][:config.store_recently_played_amount] + + Dict.Save() + + if not config.react_to_activities: + return + + debug_msg = "Started playing %s. Refreshing it." % rating_key + + # todo: cleanup debug messages for hybrid-plus + + keys_to_refresh = [] + if config.activity_mode in ["refresh", "next_episode", "hybrid", "hybrid-plus"]: + # next episode or next episode and current movie + if config.activity_mode in ["next_episode", "hybrid", "hybrid-plus"]: + plex_item = get_item(rating_key) + if not plex_item: + Log.Warn("Can't determine media type of %s, skipping" % rating_key) + return + + if get_item_kind_from_item(plex_item) == "episode": + next_ep = self.get_next_episode(rating_key) + if config.activity_mode == "hybrid-plus": + keys_to_refresh.append(rating_key) + if next_ep: + keys_to_refresh.append(next_ep.rating_key) + debug_msg = "Started playing %s. Refreshing next episode (%s, S%02iE%02i)." % \ + (rating_key, next_ep.rating_key, int(next_ep.season.index), int(next_ep.index)) + + else: + if config.activity_mode in ("hybrid", "hybrid-plus"): + keys_to_refresh.append(rating_key) + elif config.activity_mode == "refresh": + keys_to_refresh.append(rating_key) + + if keys_to_refresh: + Log.Debug(debug_msg) + Log.Debug("Refreshing %s", keys_to_refresh) + for key in keys_to_refresh: + refresh_item(key) + + def get_next_episode(self, rating_key): + plex_item = get_item(rating_key) + if not plex_item: + return + + if get_item_kind_from_item(plex_item) == "episode": + # get season + season = get_item(plex_item.season.rating_key) + if not season: + return + + # determine next episode + # next episode is in the same season + if plex_item.index < season.episode_count: + # get next ep + for ep in season.children(): + if ep.index == plex_item.index + 1: + return ep + + # it's not, try getting the first episode of the next season + else: + # get show + show = get_item(plex_item.show.rating_key) + # is there a next season? + if season.index < show.season_count: + for other_season in show.children(): + if other_season.index == season.index + 1: + next_season = other_season + for ep in next_season.children(): + if ep.index == 1: + return ep + + +activity = PlexActivityManager() diff --git a/Contents/Code/support/auth.py b/Contents/Code/support/auth.py new file mode 100644 index 000000000..e061112df --- /dev/null +++ b/Contents/Code/support/auth.py @@ -0,0 +1,42 @@ +# coding=utf-8 + + +def refresh_plex_token(): + username = Prefs["plex_username"] + password = Prefs["plex_password"] + + if not username or not password: + if "token" in Dict: + del Dict["token"] + Dict.Save() + return + + if "uuid" not in Dict: + Dict["uuid"] = String.UUID() + Dict.Save() + + current_uuid = Dict["uuid"] + + headers = { + 'X-Plex-Device-Name': 'Sub-Zero', + 'X-Plex-Product': 'Sub-Zero', + 'X-Plex-Version': '1.3.0', + 'X-Plex-Client-Identifier': "%s" % current_uuid, + } + + request = HTTP.Request("https://plex.tv/users/sign_in.json", headers=headers, + values={'user[login]': Prefs["plex_username"], 'user[password]': Prefs["plex_password"]}, immediate=True) + token = None + if request: + try: + data = JSON.ObjectFromString(request.content) + token = data["user"]["authentication_token"] + log_data = data.copy() + log_data["user"]["authentication_token"] = "xxxxxxxxxxxxxxxxxx" + Log.Debug("Data returned from plex.tv: %s", log_data) + except: + pass + if token: + Dict["token"] = token + Dict.Save() + return True diff --git a/Contents/Code/support/config.py b/Contents/Code/support/config.py new file mode 100644 index 000000000..7a4276852 --- /dev/null +++ b/Contents/Code/support/config.py @@ -0,0 +1,1202 @@ +# coding=utf-8 +import copy +import json +import os +import re +import inspect +import sys +import rarfile +import jstyleson +import datetime +import stat +import traceback +import socket +import requests + +import subliminal +import subliminal_patch +import subzero.constants +import lib +from subliminal.exceptions import ServiceUnavailable, DownloadLimitExceeded, AuthenticationError, \ + DownloadLimitPerDayExceeded +from subliminal_patch.core import is_windows_special_path +from whichdb import whichdb + +from subliminal_patch.exceptions import TooManyRequests, APIThrottled +from subzero.language import Language +from subliminal.cli import MutexLock +from subzero.lib.io import FileIO, get_viable_encoding +from subzero.lib.dict import Dicked +from subzero.lib.which import find_executable +from subzero.util import get_root_path +from subzero.constants import PLUGIN_NAME, PLUGIN_IDENTIFIER, MOVIE, SHOW, MEDIA_TYPE_TO_STRING +from subzero.prefs import get_user_prefs, update_user_prefs +from dogpile.cache.region import register_backend as register_cache_backend +from lib import Plex +from helpers import check_write_permissions, cast_bool, cast_int, mswindows + +register_cache_backend( + "subzero.cache.file", "subzero.cache_backends.file", "SZFileBackend") + +SUBTITLE_EXTS_BASE = ['utf', 'utf8', 'utf-8', 'srt', 'smi', 'rt', 'ssa', 'aqt', 'jss', 'ass', 'idx', 'sub', 'psb', + 'vtt'] +SUBTITLE_EXTS = SUBTITLE_EXTS_BASE + ["txt"] + +TEXT_SUBTITLE_EXTS = ("srt", "ass", "ssa", "vtt", "mov_text") +VIDEO_EXTS = ['3g2', '3gp', 'asf', 'asx', 'avc', 'avi', 'avs', 'bivx', 'bup', 'divx', 'dv', 'dvr-ms', 'evo', 'fli', + 'flv', + 'm2t', 'm2ts', 'm2v', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mpg', 'mts', 'nsv', 'nuv', 'ogm', 'ogv', 'tp', + 'pva', 'qt', 'rm', 'rmvb', 'sdp', 'svq3', 'strm', 'ts', 'ty', 'vdr', 'viv', 'vob', 'vp3', 'wmv', 'wpl', + 'wtv', 'xsp', 'xvid', + 'webm'] + +EXCLUDE_FN = ("subzero.ignore", ".subzero.ignore", "subzero.exclude", ".subzero.exclude", ".nosz") +INCLUDE_FN = ("subzero.include", ".subzero.include", ".sz") + +VERSION_RE = re.compile(ur'CFBundleVersion.+?([0-9\.]+)', re.DOTALL) +DEV_RE = re.compile(ur'PlexPluginDevMode.+?([01]+)', re.DOTALL) + + +def int_or_default(s, default): + try: + return int(s) + except ValueError: + return default + + +VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, DownloadLimitPerDayExceeded, + ServiceUnavailable, APIThrottled, requests.Timeout, requests.ReadTimeout, socket.timeout) + +def_timeout = (datetime.timedelta(minutes=20), "20 minutes") + +PROVIDER_THROTTLE_MAP = { + "default": { + TooManyRequests: (datetime.timedelta(hours=1), "1 hour"), + DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), + DownloadLimitPerDayExceeded: (datetime.timedelta(hours=4), "4 hours"), + ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"), + APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"), + AuthenticationError: (datetime.timedelta(hours=2), "2 hours"), + requests.Timeout: def_timeout, + socket.timeout: def_timeout, + requests.ReadTimeout: def_timeout, + }, + "opensubtitles": { + TooManyRequests: (datetime.timedelta(hours=3), "3 hours"), + DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"), + APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"), + }, + "opensubtitlescom": { + TooManyRequests: (datetime.timedelta(minutes=1), "1 minute"), + DownloadLimitExceeded: (datetime.timedelta(hours=24), "24 hours"), + }, + "addic7ed": { + DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), + TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"), + AuthenticationError: (datetime.timedelta(hours=24), "24 hours"), + } +} + + +class Config(object): + config_version = 3 + libraries_root = None + plugin_info = "" + version = None + full_version = None + plugin_log_path = None + server_log_path = None + app_support_path = None + data_path = None + data_items_path = None + universal_plex_token = None + plex_token = None + is_development = False + dbm_supported = False + pms_request_timeout = 15 + low_impact_mode = False + new_style_cache = False + pack_cache_dir = None + advanced = None + debug_i18n = False + + normal_subs = False + forced_also = False + forced_only = False + include = False + enable_channel = True + enable_agent = True + pin = None + lock_menu = False + lock_advanced_menu = False + locked = False + pin_valid_minutes = 10 + subtitle_destination_folder = None + subtitle_formats = None + max_recent_items_per_library = 200 + permissions_ok = False + missing_permissions = None + include_exclude_sz_files = False + include_exclude_paths = None + fs_encoding = None + notify_executable = None + sections = None + enabled_sections = None + remove_hi = False + remove_tags = False + fix_ocr = False + fix_common = False + fix_upper = False + reverse_rtl = False + colors = "" + chmod = None + exotic_ext = False + treat_und_as_first = False + subtitle_sub_dir = None, None + ext_match_strictness = False + default_mods = None + debug_mods = False + react_to_activities = False + activity_mode = None + no_refresh = False + plex_transcoder = None + refiner_settings = None + exact_filenames = False + only_one = False + any_language_is_enough = False + ignore_for_audio = False + ignore_subs_for_empty_audio = False + embedded_auto_extract = False + ietf_as_alpha3 = False + unrar = None + adv_cfg_path = None + use_custom_dns = None + anticaptcha_token = None + anticaptcha_cls = None + has_anticaptcha = False + mediainfo_bin = None + + store_recently_played_amount = 40 + + initialized = False + + def initialize(self): + self.libraries_root = os.path.abspath(os.path.join(get_root_path(), "..")) + self.init_libraries() + + if is_windows_special_path: + Log.Warn("The Plex metadata folder is residing inside a folder with special characters. " + "Multithreading and playback activities will be disabled.") + + self.fs_encoding = get_viable_encoding() + self.plugin_info = self.get_plugin_info() + self.is_development = self.get_dev_mode() + self.version = self.get_version() + self.full_version = u"%s %s" % (PLUGIN_NAME, self.version) + self.set_log_paths() + self.app_support_path = Core.app_support_path + self.data_path = getattr(Data, "_core").storage.data_path + self.data_items_path = os.path.join(self.data_path, "DataItems") + self.universal_plex_token = self.get_universal_plex_token() + self.plex_token = os.environ.get("PLEXTOKEN", self.universal_plex_token) + self.new_style_cache = cast_bool(Prefs['new_style_cache']) + self.pack_cache_dir = self.get_pack_cache_dir() + try: + self.migrate_prefs() + except: + Log.Exception("Catastrophic failure when running prefs migration") + + subzero.constants.DEFAULT_TIMEOUT = lib.DEFAULT_TIMEOUT = self.pms_request_timeout = \ + min(cast_int(Prefs['pms_request_timeout'], 15), 45) + self.low_impact_mode = cast_bool(Prefs['low_impact_mode']) + self.advanced = self.get_advanced_config() + self.debug_i18n = self.advanced.debug_i18n + + os.environ["SZ_USER_AGENT"] = self.get_user_agent() + os.environ["ANTICAPTCHA_ACCOUNT_KEY"] = self.anticaptcha_token = str(Prefs["anticaptcha.api_key"]) or "" + acs = str(Prefs["anticaptcha.service"]) + if acs and acs != "none": + os.environ["ANTICAPTCHA_CLASS"] = self.anticaptcha_cls = acs + self.has_anticaptcha = self.anticaptcha_token and self.anticaptcha_cls + + self.setup_proxies() + self.set_plugin_mode() + self.set_plugin_lock() + self.set_activity_modes() + self.parse_rename_mode() + + self.normal_subs = Prefs["subtitles.when"] != "Never" + self.forced_also = self.normal_subs and Prefs["subtitles.when_forced"] != "Never" + self.forced_only = not self.normal_subs and Prefs["subtitles.when_forced"] != "Never" + self.include = \ + Prefs["subtitles.include_exclude_mode"] == "disable SZ for all items by default, use include lists" + self.subtitle_destination_folder = self.get_subtitle_destination_folder() + self.subtitle_formats = self.get_subtitle_formats() + self.max_recent_items_per_library = int_or_default(Prefs["scheduler.max_recent_items_per_library"], 2000) + self.sections = list(Plex["library"].sections()) + self.missing_permissions = [] + self.include_exclude_sz_files = cast_bool(Prefs["subtitles.include_exclude_fs"]) + self.include_exclude_paths = self.parse_include_exclude_paths() + self.enabled_sections = self.check_enabled_sections() + self.permissions_ok = self.check_permissions() + self.notify_executable = self.check_notify_executable() + self.remove_hi = cast_bool(Prefs['subtitles.remove_hi']) + self.remove_tags = cast_bool(Prefs['subtitles.remove_tags']) + self.fix_ocr = cast_bool(Prefs['subtitles.fix_ocr']) + self.fix_common = cast_bool(Prefs['subtitles.fix_common']) + self.fix_upper = cast_bool(Prefs['subtitles.fix_only_uppercase']) + self.reverse_rtl = cast_bool(Prefs['subtitles.reverse_rtl']) + self.colors = Prefs['subtitles.colors'] if Prefs['subtitles.colors'] != "don't change" else None + self.chmod = self.check_chmod() + self.exotic_ext = cast_bool(Prefs["subtitles.scan.exotic_ext"]) + self.treat_und_as_first = cast_bool(Prefs["subtitles.language.treat_und_as_first"]) + self.subtitle_sub_dir = self.get_subtitle_sub_dir() + self.ext_match_strictness = self.determine_ext_sub_strictness() + self.default_mods = self.get_default_mods() + self.debug_mods = cast_bool(Prefs['log_debug_mods']) + self.no_refresh = os.environ.get("SZ_NO_REFRESH", False) + self.plex_transcoder = self.get_plex_transcoder() + self.only_one = cast_bool(Prefs['subtitles.only_one']) + self.any_language_is_enough = Prefs['subtitles.any_language_is_enough'] + self.ignore_for_audio = self.ignore_subs_for_audio() + self.embedded_auto_extract = cast_bool(Prefs["subtitles.embedded.autoextract"]) + self.ietf_as_alpha3 = cast_bool(Prefs["subtitles.language.ietf_normalize"]) + self.use_custom_dns = self.parse_custom_dns() + if not self.advanced.dont_use_mediainfo_mp4: + self.mediainfo_bin = self.advanced.mediainfo_bin or find_executable("mediainfo") + self.initialized = True + + def migrate_prefs(self): + config_version = 0 if "config_version" not in Dict else Dict["config_version"] + if config_version < self.config_version: + user_prefs = get_user_prefs(Prefs, Log) + if user_prefs: + update_prefs = {} + for i in range(config_version, self.config_version): + version = i+1 + func = "migrate_prefs_to_%i" % version + if hasattr(self, func): + Log.Info("Migrating user prefs to version %i" % version) + try: + mig_result = getattr(self, func)(user_prefs, from_version=config_version, + to_version=version, + current_version=self.config_version, + migrated_prefs=update_prefs) + update_prefs.update(mig_result) + Dict["config_version"] = version + Dict.Save() + Log.Info("User prefs migrated to version %i" % version) + except: + Log.Exception("User prefs migration from %i to %i failed" % (self.config_version, version)) + break + + if update_prefs: + update_user_prefs(update_prefs, Prefs, Log) + else: + Dict["config_version"] = self.config_version + Dict.Save() + + def migrate_prefs_to_1(self, user_prefs, **kwargs): + update_prefs = {} + if "subtitles.only_foreign" in user_prefs and user_prefs["subtitles.only_foreign"] == "true": + update_prefs["subtitles.when_forced"] = "1" + update_prefs["subtitles.when"] = "0" + + return update_prefs + + def migrate_prefs_to_2(self, user_prefs, **kwargs): + update_prefs = {} + if "subtitles.ignore_fs" in user_prefs and user_prefs["subtitles.ignore_fs"] == "true": + update_prefs["subtitles.include_exclude_fs"] = "true" + + if "subtitles.ignore_paths" in user_prefs and user_prefs["subtitles.ignore_paths"]: + update_prefs["subtitles.include_exclude_paths"] = user_prefs["subtitles.ignore_paths"] + + return update_prefs + + def migrate_prefs_to_3(self, user_prefs, **kwargs): + if config.new_style_cache: + self.init_cache() + try: + subliminal.region.backend.clear() + except: + pass + return {} + + def init_libraries(self): + try_executables = [] + custom_unrar = os.environ.get("SZ_UNRAR_TOOL") + if custom_unrar: + if os.path.isfile(custom_unrar): + try_executables.append(custom_unrar) + + unrar_exe = None + if Core.runtime.os == "Windows": + unrar_exe = os.path.abspath(os.path.join(self.libraries_root, "Windows", "i386", "UnRAR", "UnRAR.exe")) + + elif Core.runtime.os == "MacOSX": + unrar_exe = os.path.abspath(os.path.join(self.libraries_root, "MacOSX", "i386", "UnRAR", "unrar")) + + elif Core.runtime.os == "Linux": + unrar_exe = os.path.abspath(os.path.join(self.libraries_root, "Linux", Core.runtime.cpu, "UnRAR", "unrar")) + + if unrar_exe and os.path.isfile(unrar_exe): + try_executables.append(unrar_exe) + + try_executables.append("unrar") + + for exe in try_executables: + rarfile.UNRAR_TOOL = exe + rarfile.ORIG_UNRAR_TOOL = exe + if os.path.isfile(exe) and not os.access(exe, os.X_OK): + st = os.stat(exe) + try: + Log.Debug("setting generic executable permissions for %s", exe) + # fixme: too broad? + os.chmod(exe, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + except: + Log.Debug("failed setting generic executable permissions for %s: %s", exe, traceback.format_exc()) + try: + Log.Debug("setting executable permissions for %s", exe) + os.chmod(exe, st.st_mode | stat.S_IEXEC) + except: + Log.Debug("failed setting executable permissions for %s: %s", exe, traceback.format_exc()) + try: + rarfile.custom_check([rarfile.UNRAR_TOOL], True) + except: + Log.Debug("custom check failed for: %s", exe) + continue + + rarfile.OPEN_ARGS = rarfile.ORIG_OPEN_ARGS + rarfile.EXTRACT_ARGS = rarfile.ORIG_EXTRACT_ARGS + rarfile.TEST_ARGS = rarfile.ORIG_TEST_ARGS + Log.Info("Using UnRAR from: %s", exe) + self.unrar = exe + return + + Log.Warn("UnRAR not found") + + def init_cache(self): + if self.new_style_cache: + subliminal.region.configure('subzero.cache.file', expiration_time=datetime.timedelta(days=180), + arguments={'appname': "sz_cache", + 'app_cache_dir': self.data_path}, + replace_existing_backend=True) + Log.Info("Using new style file based cache!") + return + + names = ['dbhash', 'gdbm', 'dbm'] + dbfn = None + self.dbm_supported = False + + # try importing dbm modules + if Core.runtime.os != "Windows": + impawrt = None + try: + impawrt = getattr(sys.modules['__main__'], "__builtins__").get("__import__") + except: + pass + + if impawrt: + for name in names: + try: + impawrt(name) + except: + continue + if not self.dbm_supported: + self.dbm_supported = name + break + + if self.dbm_supported: + # anydbm checks; try guessing the format and importing the correct module + dbfn = os.path.join(config.data_items_path, 'subzero.dbm') + db_which = whichdb(dbfn) + if db_which is not None and db_which != "": + try: + impawrt(db_which) + except ImportError: + self.dbm_supported = False + + if self.dbm_supported: + try: + subliminal.region.configure('dogpile.cache.dbm', expiration_time=datetime.timedelta(days=30), + arguments={'filename': dbfn, + 'lock_factory': MutexLock}, + replace_existing_backend=True) + Log.Info("Using file based cache!") + return + except: + self.dbm_supported = False + + Log.Warn("Not using file based cache!") + subliminal.region.configure('dogpile.cache.memory', replace_existing_backend=True) + + def sync_cache(self): + if not self.new_style_cache: + return + Log.Debug("Syncing cache") + subliminal.region.backend.sync() + + def get_pack_cache_dir(self): + pack_cache_dir = os.path.join(config.data_path, "pack_cache") + if not os.path.isdir(pack_cache_dir): + os.makedirs(pack_cache_dir) + + return pack_cache_dir + + def get_advanced_config(self): + paths = [] + if Prefs['path_to_advanced_settings']: + paths = [ + Prefs['path_to_advanced_settings'], + os.path.join(Prefs['path_to_advanced_settings'], "advanced_settings.json") + ] + + paths.append(os.path.join(config.data_path, "advanced_settings.json")) + + for path in paths: + if os.path.isfile(path): + data = FileIO.read(path, "r") + + d = Dicked(**jstyleson.loads(data)) + self.adv_cfg_path = path + Log.Info(u"Using advanced settings from: %s", path) + return d + + return Dicked() + + def set_log_paths(self): + # find log handler + for handler in Core.log.handlers: + cls_name = getattr(getattr(handler, "__class__"), "__name__") + if cls_name in ('FileHandler', 'RotatingFileHandler', 'TimedRotatingFileHandler'): + plugin_log_file = handler.baseFilename + if cls_name in ("RotatingFileHandler", "TimedRotatingFileHandler"): + handler.backupCount = int_or_default(Prefs['log_rotate_keep'], 5) + + if os.path.isfile(os.path.realpath(plugin_log_file)): + self.plugin_log_path = plugin_log_file + + if plugin_log_file: + server_log_file = os.path.realpath(os.path.join(plugin_log_file, "../../Plex Media Server.log")) + if os.path.isfile(server_log_file): + self.server_log_path = server_log_file + + def get_universal_plex_token(self): + # thanks to: https://forums.plex.tv/discussion/247136/read-current-x-plex-token-in-an-agent-ensure-that-a-http-request-gets-executed-exactly-once#latest + pref_path = os.path.join(self.app_support_path, "Preferences.xml") + if os.path.exists(pref_path): + try: + global_prefs = Core.storage.load(pref_path) + return XML.ElementFromString(global_prefs).xpath('//Preferences/@PlexOnlineToken')[0] + except: + Log.Warn("Couldn't determine Plex Token") + else: + Log.Warn("Did NOT find Preferences file - most likely Windows OS. Otherwise please check logfile and hierarchy.") + + # fixme: windows + + def set_plugin_mode(self): + self.enable_agent = True + self.enable_channel = True + + # any provider enabled? + if not self.providers_enabled: + self.enable_agent = False + self.enable_channel = False + Log.Warn("No providers enabled, disabling agent and interface!") + return + + if Prefs["plugin_mode2"] == "only agent": + self.enable_channel = False + elif Prefs["plugin_mode2"] == "only interface": + self.enable_agent = False + + def set_plugin_lock(self): + if Prefs["plugin_pin_mode2"] in ("interface", "advanced menu"): + # check pin + pin = Prefs["plugin_pin"] + if not pin or not len(pin): + Log.Warn("PIN enabled but not set, disabling PIN!") + return + + pin = pin.strip() + try: + int(pin) + except ValueError: + Log.Warn("PIN has to be an integer (0-9)") + self.pin = pin + self.lock_advanced_menu = Prefs["plugin_pin_mode2"] == "advanced menu" + self.lock_menu = Prefs["plugin_pin_mode2"] == "interface" + + try: + self.pin_valid_minutes = int(Prefs["plugin_pin_valid_for"].strip()) + except ValueError: + pass + + @property + def pin_correct(self): + if isinstance(Dict["pin_correct_time"], datetime.datetime) \ + and Dict["pin_correct_time"] + datetime.timedelta( + minutes=self.pin_valid_minutes) > datetime.datetime.now(): + return True + + def refresh_permissions_status(self): + self.permissions_ok = self.check_permissions() + + def check_permissions(self): + if not cast_bool(Prefs["subtitles.save.filesystem"]) or not cast_bool(Prefs["check_permissions"]): + return True + + self.missing_permissions = [] + use_include_exclude_fs = self.include_exclude_sz_files + all_permissions_ok = True + for section in self.sections: + if section.key not in self.enabled_sections: + continue + + title = section.title + for location in section: + path_str = location.path + if isinstance(path_str, unicode): + path_str = path_str.encode(self.fs_encoding) + + if not os.path.exists(path_str): + continue + + if use_include_exclude_fs: + # check whether we've got an ignore file inside the section path + if not self.is_physically_wanted(path_str): + continue + + if not self.is_path_wanted(path_str): + # is the path in our ignored paths setting? + continue + + # section not ignored, check for write permissions + if not check_write_permissions(path_str): + # not enough permissions + self.missing_permissions.append((title, location.path)) + all_permissions_ok = False + + return all_permissions_ok + + def get_version(self): + return self.get_bare_version() + ("" if not self.is_development else " DEV") + + def get_bare_version(self): + result = VERSION_RE.search(self.plugin_info) + + if result: + return result.group(1) + return "2.x.x.x" + + def get_user_agent(self): + return "Sub-Zero/%s" % (self.get_bare_version() + ("" if not self.is_development else "-dev")) + + def get_dev_mode(self): + dev = DEV_RE.search(self.plugin_info) + if dev and dev.group(1) == "1": + return True + + def get_plugin_info(self): + curDir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) + info_file_path = os.path.abspath(os.path.join(curDir, "..", "..", "Info.plist")) + return FileIO.read(info_file_path) + + def parse_include_exclude_paths(self): + paths = Prefs["subtitles.include_exclude_paths"] + if paths and paths != "None": + try: + return [p for p in [path.strip() for path in paths.split(",")] + if p != "None" and os.path.exists(p)] + except: + Log.Error("Couldn't parse your include/exclude paths settings: %s" % paths) + return [] + + def is_physically_wanted(self, folder, ref_fn=None): + # check whether we've got an ignore file inside the path + ret_val = self.include + ref_list = INCLUDE_FN if self.include else EXCLUDE_FN + for ifn in ref_list: + if os.path.isfile(os.path.join(folder, ifn)): + if ref_fn: + Log.Info(u'%s "%s" because "%s" exists in "%s"', "Including" if self.include else "Ignoring", + ref_fn, ifn, folder) + else: + Log.Info(u'%s "%s" because "%s" exists', "Including" if self.include else "Ignoring", folder, ifn) + + return ret_val + + return not ret_val + + def is_path_wanted(self, fn): + ret_val = self.include + for path in self.include_exclude_paths: + if fn.startswith(path): + return ret_val + return not ret_val + + def check_notify_executable(self): + fn = Prefs["notify_executable"] + if not fn: + return + + got_args = "%(" in fn + if got_args: + first_arg_pos = fn.index("%(") + exe_fn = fn[:first_arg_pos].strip() + arguments = [arg.strip() for arg in fn[first_arg_pos:].split()] + else: + exe_fn = fn + arguments = [] + + if os.path.isfile(exe_fn) and os.access(exe_fn, os.X_OK): + return exe_fn, arguments + + # try finding the executable itself, the path might contain spaces and there might have been other arguments + fn_split = exe_fn.split(u" ") + tmp_exe_fn = fn_split[0] + + for offset in range(1, len(fn_split)+1): + if os.path.isfile(tmp_exe_fn) and os.access(tmp_exe_fn, os.X_OK): + exe_fn = tmp_exe_fn.strip() + arguments = [arg.strip() for arg in fn_split[offset:]] + arguments + return exe_fn, arguments + + tmp_exe_fn = u" ".join(fn_split[:offset+1]) + + Log.Error("Notify executable not existing or not executable: %s" % exe_fn) + + def refresh_enabled_sections(self): + self.enabled_sections = self.check_enabled_sections() + + def check_enabled_sections(self): + enabled_for_primary_agents = {"movie": [], "show": []} + enabled_sections = {} + + legacy_agents = { + "com.plexapp.agents.thetvdb": [SHOW], + "com.plexapp.agents.thetvdbdvdorder": [SHOW], + "com.plexapp.agents.hama": [SHOW, MOVIE], + "com.plexapp.agents.themoviedb": [SHOW, MOVIE], + "com.plexapp.agents.imdb": [SHOW, MOVIE], + } + + # find which agents we're enabled for + for agent in Plex.agents(): + #if not agent.primary: + # continue + if agent.identifier not in legacy_agents: + continue + + #media_types = [t.media_type for t in list(agent.media_types)] + media_types = legacy_agents[agent.identifier] + [] + + # the new movie agent doesn't populate its media types, workaround + if not media_types and agent.identifier == "tv.plex.agents.movie": + media_types = [MOVIE] + + for media_type in media_types: + if media_type in (MOVIE, SHOW): + related_agents = Plex.primary_agent(agent.identifier, media_type) + for a in related_agents: + if a.identifier == PLUGIN_IDENTIFIER and a.enabled: + enabled_for_primary_agents[MEDIA_TYPE_TO_STRING[media_type]].append(agent.identifier) + + # find the libraries that use them + for library in self.sections: + if library.agent in enabled_for_primary_agents.get(library.type, []): + enabled_sections[library.key] = library + + Log.Debug(u"I'm enabled for: %s" % [lib.title for key, lib in enabled_sections.iteritems()]) + return enabled_sections + + def lang_str_to_list(self, s, l): + if len(s) and s != "None": + for lang in s.split(u","): + lang = lang.strip() + if lang == "NULL": + l.append(lang) + continue + try: + real_lang = Language.fromietf(lang) + except: + try: + real_lang = Language.fromname(lang) + except: + continue + l.append(real_lang) + + # Prepare a list of languages we want subs for + def get_lang_list(self, provider=None, ordered=False): + # advanced settings + if provider and self.advanced.providers and provider in self.advanced.providers: + adv_languages = self.advanced.providers[provider].get("languages", None) + if adv_languages: + adv_out = set() + for adv_lang in adv_languages: + adv_lang = adv_lang.strip() + try: + real_lang = Language.fromietf(adv_lang) + except: + try: + real_lang = Language.fromname(adv_lang) + except: + continue + adv_out.update({real_lang}) + + # fallback to default languages if no valid language was found in advanced settings + if adv_out: + return adv_out + + l = [Language.fromietf(Prefs["langPref1a"])] + lang_custom = Prefs["langPrefCustom"].strip() + + if Prefs['subtitles.only_one']: + return set(l) if not ordered else l + + if Prefs["langPref2a"] != "None": + try: + l.append(Language.fromietf(Prefs["langPref2a"])) + except: + pass + + if Prefs["langPref3a"] != "None": + try: + l.append(Language.fromietf(Prefs["langPref3a"])) + except: + pass + + self.lang_str_to_list(lang_custom, l) + if "NULL" in l: + l.remove("NULL") + + if self.forced_also: + if Prefs["subtitles.when_forced"] == "Always": + for lang in list(l): + l.append(Language.rebuild(lang, forced=True)) + + else: + for (setting, index) in (("Only for Subtitle Language (1)", 0), + ("Only for Subtitle Language (2)", 1), + ("Only for Subtitle Language (3)", 2)): + if Prefs["subtitles.when_forced"] == setting: + try: + l.append(Language.rebuild(list(l)[index], forced=True)) + break + except: + pass + + elif self.forced_only: + for lang in l: + lang.forced = True + + if not self.normal_subs: + for lang in l[:]: + if not lang.forced: + l.remove(lang) + + return set(l) if not ordered else l + + lang_list = property(get_lang_list) + + def ignore_subs_for_audio(self): + c = Prefs['subtitles.ignore_for_audio'].strip() + l = [] + + self.lang_str_to_list(c, l) + if "NULL" in l: + l.remove("NULL") + self.ignore_subs_for_empty_audio = True + return l + + def get_subtitle_destination_folder(self): + if not Prefs["subtitles.save.filesystem"]: + return + + fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if \ + Prefs["subtitles.save.subFolder.Custom"] else None + return fld_custom or ( + Prefs["subtitles.save.subFolder"] if Prefs["subtitles.save.subFolder"] != "current folder" else None) + + def get_subtitle_formats(self): + formats = Prefs["subtitles.save.formats"] + out = [] + if "SRT" in formats: + out.append("srt") + if "VTT" in formats: + out.append("vtt") + return out + + @property + def providers_by_prefs(self): + return {'opensubtitlescom': cast_bool(Prefs['provider.opensubtitles.enabled']), + # 'thesubdb': Prefs['provider.thesubdb.enabled'], + 'podnapisi': cast_bool(Prefs['provider.podnapisi.enabled']), + 'napisy24': cast_bool(Prefs['provider.napisy24.enabled']), + 'titlovi': cast_bool(Prefs['provider.titlovi.enabled']), + 'addic7ed': cast_bool(Prefs['provider.addic7ed.enabled']) and self.has_anticaptcha, + 'tvsubtitles': cast_bool(Prefs['provider.tvsubtitles.enabled']), + 'legendastv': cast_bool(Prefs['provider.legendastv.enabled']), + 'napiprojekt': cast_bool(Prefs['provider.napiprojekt.enabled']), + 'hosszupuska': cast_bool(Prefs['provider.hosszupuska.enabled']), + 'supersubtitles': cast_bool(Prefs['provider.supersubtitles.enabled']), + 'shooter': False, + 'subscene': cast_bool(Prefs['provider.subscene.enabled']), + 'argenteam': cast_bool(Prefs['provider.argenteam.enabled']), + 'subscenter': False, + 'assrt': cast_bool(Prefs['provider.assrt.enabled']), + 'bsplayer': cast_bool(Prefs['provider.bsplayer.enabled']), + 'ktuvit': cast_bool(Prefs['provider.ktuvit.enabled']), + 'wizdom': cast_bool(Prefs['provider.wizdom.enabled']), + } + + @property + def providers_enabled(self): + return filter(lambda prov: self.providers_by_prefs[prov], self.providers_by_prefs) + + def get_providers(self, media_type="series"): + providers = self.providers_by_prefs + providers_by_prefs = copy.deepcopy(providers) + + # disable subscene for movies by default + if media_type == "movies": + providers["subscene"] = False + + # ditch non-forced-subtitles-reporting providers + providers_forced_off = {} + if self.forced_only: + providers["addic7ed"] = False + providers["tvsubtitles"] = False + providers["legendastv"] = False + providers["napiprojekt"] = False + providers["shooter"] = False + providers["hosszupuska"] = False + providers["supersubtitles"] = False + providers["titlovi"] = False + providers["argenteam"] = False + providers["assrt"] = False + providers["subscene"] = False + providers["napisy24"] = False + providers["bsplayer"] = False + providers["ktuvit"] = False + providers["wizdom"] = False + providers_forced_off = dict(providers) + + if not self.unrar and providers["legendastv"]: + providers["legendastv"] = False + Log.Info("Disabling LegendasTV, because UnRAR wasn't found") + + # advanced settings + if media_type and self.advanced.providers: + for provider, data in self.advanced.providers.iteritems(): + if provider not in providers or not providers_by_prefs[provider] or provider in providers_forced_off: + continue + + if data["enabled_for"] is not None: + providers[provider] = media_type in data["enabled_for"] + + if "provider_throttle" not in Dict: + Dict["provider_throttle"] = {} + + changed = False + for provider, enabled in dict(providers).iteritems(): + reason, until, throttle_desc = Dict["provider_throttle"].get(provider, (None, None, None)) + if reason: + now = datetime.datetime.now() + if now < until: + Log.Info("Not using %s until %s, because of: %s", provider, + until.strftime("%y/%m/%d %H:%M"), reason) + providers[provider] = False + else: + Log.Info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason) + del Dict["provider_throttle"][provider] + changed = True + + if changed: + Dict.Save() + + return filter(lambda prov: providers[prov], providers) + + providers = property(get_providers) + + def get_provider_settings(self): + os_use_https = self.advanced.providers.opensubtitles.get("use_https", True) + os_skip_wrong_fps = self.advanced.providers.opensubtitles.get("skip_wrong_fps", True) + + provider_settings = {'addic7ed': {'username': Prefs['provider.addic7ed.username'], + 'password': Prefs['provider.addic7ed.password'], + 'is_vip': cast_bool(Prefs['provider.addic7ed.is_vip']), + }, + 'opensubtitlescom': {'username': Prefs['provider.opensubtitles.username'], + 'password': Prefs['provider.opensubtitles.password'], + #'use_tag_search': self.exact_filenames, + #'only_foreign': self.forced_only, + #'also_foreign': self.forced_also, + #'is_vip': cast_bool(Prefs['provider.opensubtitles.is_vip']), + #'use_ssl': os_use_https, + #'timeout': self.advanced.providers.opensubtitles.timeout or 15, + #'skip_wrong_fps': os_skip_wrong_fps, + 'use_hash': cast_bool(Prefs['provider.opensubtitles.use_hash']), + 'include_ai_translated': True, + 'api_key': Prefs['provider.opensubtitles.api_key'], + }, + 'podnapisi': { + 'only_foreign': self.forced_only, + 'also_foreign': self.forced_also, + }, + 'titlovi': { + 'username': Prefs['provider.titlovi.username'], + 'password': Prefs['provider.titlovi.password'], + }, + 'napisy24': { + 'username': Prefs['provider.napisy24.username'], + 'password': Prefs['provider.napisy24.password'], + }, + 'subscene': { + 'only_foreign': self.forced_only, + 'username': Prefs['provider.subscene.username'], + 'password': Prefs['provider.subscene.password'], + }, + 'legendastv': {'username': Prefs['provider.legendastv.username'], + 'password': Prefs['provider.legendastv.password'], + }, + 'assrt': {'token': Prefs['provider.assrt.token'], }, + 'ktuvit': { + 'username': Prefs['provider.ktuvit.username'], + 'password': Prefs['provider.ktuvit.password'], + }, + } + + return provider_settings + + provider_settings = property(get_provider_settings) + + def provider_throttle(self, name, exception): + """ + throttle a provider :name: for X hours based on the :exception: type + :param name: + :param exception: + :return: + """ + cls = getattr(exception, "__class__") + cls_name = getattr(cls, "__name__") + if cls not in VALID_THROTTLE_EXCEPTIONS: + for valid_cls in VALID_THROTTLE_EXCEPTIONS: + if isinstance(cls, valid_cls): + cls = valid_cls + + throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \ + PROVIDER_THROTTLE_MAP["default"].get(cls, None) + + if throttle_data: + throttle_delta, throttle_description = throttle_data + else: + throttle_delta, throttle_description = datetime.timedelta(minutes=10), "10 minutes" + + if "provider_throttle" not in Dict: + Dict["provider_throttle"] = {} + + throttle_until = datetime.datetime.now() + throttle_delta + Dict["provider_throttle"][name] = (cls_name, throttle_until, throttle_description) + + Log.Info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description, + throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message) + Dict.Save() + + @property + def provider_pool(self): + if cast_bool(Prefs['providers.multithreading']): + return subliminal_patch.core.SZAsyncProviderPool + return subliminal_patch.core.SZProviderPool + + def check_chmod(self): + val = Prefs["subtitles.save.chmod"] + if not val or not len(val): + return + + wrong_chmod = False + if len(val) != 4: + wrong_chmod = True + + try: + return int(val, 8) + except ValueError: + wrong_chmod = True + + if wrong_chmod: + Log.Warn("Chmod setting ignored, please use only 4-digit integers with leading 0 (e.g.: 775)") + + def get_subtitle_sub_dir(self): + """ + + :return: folder, is_absolute + """ + if not cast_bool(Prefs['subtitles.save.filesystem']): + return None, None + + if Prefs["subtitles.save.subFolder.Custom"]: + return Prefs["subtitles.save.subFolder.Custom"], os.path.isabs(Prefs["subtitles.save.subFolder.Custom"]) + + if Prefs["subtitles.save.subFolder"] == "current folder": + return ".", False + + return Prefs["subtitles.save.subFolder"], False + + def determine_ext_sub_strictness(self): + val = Prefs["subtitles.scan.filename_strictness"] + if val == "any": + return "any" + elif val.startswith("loose"): + return "loose" + return "strict" + + def get_default_mods(self): + mods = [] + if self.remove_hi: + mods.append("remove_HI") + if self.remove_tags: + mods.append("remove_tags") + if self.fix_ocr: + mods.append("OCR_fixes") + if self.fix_common: + mods.append("common") + if self.fix_upper: + mods.append("fix_uppercase") + if self.colors: + mods.append("color(name=%s)" % self.colors) + if self.reverse_rtl: + mods.append("reverse_rtl") + + return mods + + def setup_proxies(self): + proxy = Prefs["proxy"] + if proxy: + os.environ["SZ_HTTP_PROXY"] = proxy.strip() + Log.Debug("Using HTTP Proxy: %s", proxy) + + def set_activity_modes(self): + val = Prefs["activity.on_playback"] + if val == "never": + self.react_to_activities = False + return + + self.react_to_activities = True + if val == "current media item": + self.activity_mode = "refresh" + elif val == "hybrid: current item or next episode": + self.activity_mode = "hybrid" + elif val == "hybrid-plus: current item and next episode": + self.activity_mode = "hybrid-plus" + else: + self.activity_mode = "next_episode" + + def get_plex_transcoder(self): + paths = [] + base_path = os.environ.get("PLEX_MEDIA_SERVER_HOME", None) + if base_path: + paths.append(base_path) + + bundle_path = os.environ.get("PLEXBUNDLEDPLUGINSPATH", None) + if bundle_path: + paths.append(os.path.normpath(os.path.join(bundle_path, "..", ".."))) + + paths.append(self.app_support_path) + + bns = [] + if sys.platform == "darwin": + bns.append(("MacOS", "Plex Transcoder")) + elif mswindows: + bns = [("plextranscoder.exe",), ("plex transcoder.exe",)] + else: + bns.append(("Plex Transcoder",)) + + for path in paths: + for bn in bns: + fn = os.path.join(path, *bn) + + if os.path.isfile(fn): + return fn + + # look inside Resources folder as fallback, as well + for vbn in ("Plex Transcoder", "plextranscoder.exe", "plex transcoder.exe"): + fn = os.path.join(path, "Resources", vbn) + if os.path.isfile(fn): + return fn + + def parse_rename_mode(self): + # fixme: exact_filenames should be determined via callback combined with info about the current video + # (original_name) + + mode = str(Prefs["media_rename1"]) + self.refiner_settings = {} + + if cast_bool(Prefs['use_file_info_file']): + self.refiner_settings["file_info_file"] = True + self.exact_filenames = True + + if mode == "none of the above": + return + + elif mode == "Symlink to original file": + self.refiner_settings["symlinks"] = True + self.exact_filenames = True + return + + elif mode == "I keep the original filenames": + self.exact_filenames = True + return + + if mode in ("Filebot", "Sonarr/Radarr/Filebot"): + self.refiner_settings["filebot"] = True + + if mode in ("Sonarr/Radarr (fill api info below)", "Sonarr/Radarr/Filebot"): + if Prefs["drone_api.sonarr.url"] and Prefs["drone_api.sonarr.api_key"]: + self.refiner_settings["sonarr"] = { + "base_url": Prefs["drone_api.sonarr.url"], + "api_key": Prefs["drone_api.sonarr.api_key"], + } + if self.advanced.refiners.sonarr: + self.refiner_settings["sonarr"].update(self.advanced.refiners.sonarr) + + self.exact_filenames = True + + if Prefs["drone_api.radarr.url"] and Prefs["drone_api.radarr.api_key"]: + self.refiner_settings["radarr"] = { + "base_url": Prefs["drone_api.radarr.url"], + "api_key": Prefs["drone_api.radarr.api_key"] + } + if self.advanced.refiners.radarr: + self.refiner_settings["radarr"].update(self.advanced.refiners.radarr) + + self.exact_filenames = True + + @property + def text_based_formats(self): + return self.advanced.text_subtitle_formats or TEXT_SUBTITLE_EXTS + + def parse_custom_dns(self): + custom_dns = Prefs['use_custom_dns2'].strip() + os.environ["dns_resolvers"] = "" + + if custom_dns and custom_dns != "system": + ips = filter(lambda x: x, [d.strip() for d in custom_dns.split(",")]) + if ips: + os.environ["dns_resolvers"] = json.dumps(ips) + return os.environ["dns_resolvers"] + + def init_subliminal_patches(self): + # configure custom subtitle destination folders for scanning pre-existing subs + Log.Debug("Patching subliminal ...") + dest_folder = self.subtitle_destination_folder + subliminal_patch.core.CUSTOM_PATHS = [dest_folder] if dest_folder else [] + subliminal_patch.core.INCLUDE_EXOTIC_SUBS = self.exotic_ext + + subliminal_patch.core.DOWNLOAD_TRIES = int(Prefs['subtitles.try_downloads']) + subliminal.score.episode_scores["addic7ed_boost"] = int(Prefs['provider.addic7ed.boost_by2']) + + +config = Config() +config.initialize() diff --git a/Contents/Code/support/data.py b/Contents/Code/support/data.py new file mode 100644 index 000000000..bef55b66f --- /dev/null +++ b/Contents/Code/support/data.py @@ -0,0 +1,89 @@ +# coding=utf-8 +import traceback + + +def dispatch_migrate(): + try: + migrate() + except: + Log.Error("Migration failed: %s" % traceback.format_exc()) + del Dict["subs"] + Dict.Save() + + +def migrate(): + """ + some Dict/Data migrations here, no need for a more in-depth migration path for now + :return: + """ + + # migrate subtitle history from Dict to Data + if "history" in Dict and Dict["history"].get("history_items"): + Log.Debug("Running migration for history data") + from support.history import get_history + history = get_history() + + for item in reversed(Dict["history"]["history_items"]): + history.add(item.item_title, item.rating_key, item.section_title, subtitle=item.subtitle, mode=item.mode, + time=item.time) + + del Dict["history"] + history.destroy() + Dict.Save() + + # migrate subtitle storage from Dict to Data + if "subs" in Dict: + from support.storage import get_subtitle_storage + from subzero.subtitle_storage import StoredSubtitle + from support.plex_media import get_item + + subtitle_storage = get_subtitle_storage() + + for video_id, parts in Dict["subs"].iteritems(): + try: + item = get_item(video_id) + except: + continue + + if not item: + continue + + stored_subs = subtitle_storage.load_or_new(item) + stored_subs.version = 1 + + Log.Debug(u"Migrating %s" % video_id) + + stored_any = False + for part_id, lang_dict in parts.iteritems(): + part_id = str(part_id) + Log.Debug(u"Migrating %s, %s" % (video_id, part_id)) + + for lang, subs in lang_dict.iteritems(): + lang = str(lang) + if "current" in subs: + current_key = subs["current"] + provider_name, subtitle_id = current_key + sub = subs.get(current_key) + if sub and sub.get("title") and sub.get("mode"): # ditch legacy data without sufficient info + stored_subs.title = sub["title"] + new_sub = StoredSubtitle(sub["score"], sub["storage"], sub["hash"], provider_name, + subtitle_id, date_added=sub["date_added"], mode=sub["mode"]) + + if part_id not in stored_subs.parts: + stored_subs.parts[part_id] = {} + + if lang not in stored_subs.parts[part_id]: + stored_subs.parts[part_id][lang] = {} + + Log.Debug(u"Migrating %s, %s, %s" % (video_id, part_id, current_key)) + + stored_subs.parts[part_id][lang][current_key] = new_sub + stored_subs.parts[part_id][lang]["current"] = current_key + stored_any = True + + if stored_any: + subtitle_storage.save(stored_subs) + + subtitle_storage.destroy() + del Dict["subs"] + Dict.Save() diff --git a/Contents/Code/support/download.py b/Contents/Code/support/download.py new file mode 100644 index 000000000..ea68f705e --- /dev/null +++ b/Contents/Code/support/download.py @@ -0,0 +1,152 @@ +# coding=utf-8 +import os + +from subzero.language import Language + +import subliminal_patch as subliminal + +from support.config import config +from support.helpers import audio_streams_match_languages +from subliminal_patch import compute_score +from support.plex_media import get_blacklist_from_part_map +from subzero.video import refine_video +from support.storage import get_pack_data, store_pack_data + + +def get_missing_languages(video, part): + languages_list = config.get_lang_list(ordered=True) + languages = set(languages_list) + ignore_langs = set(config.ignore_for_audio) + valid_langs_in_media = set() + + if Prefs["subtitles.when"] != "Always": + valid_langs_in_media = audio_streams_match_languages(video, languages_list) + languages = languages.difference(valid_langs_in_media) + if languages: + Log.Debug("Languages missing after taking the audio streams into account: %s" % languages) + + if valid_langs_in_media and not languages: + Log.Debug("Skipping subtitle search for %s, audio streams are in correct language(s)", + video) + return set() + + if config.ignore_subs_for_empty_audio and not video.audio_languages: + Log.Debug("Skipping subtitle search for %s, ignoring as no audio stream exists", video) + return set() + + inter = ignore_langs.intersection(video.audio_languages) + if ignore_langs and inter: + Log.Debug("Skipping subtitle search for %s, ignoring due to existing audio streams: %s", video, inter) + return set() + + # should we treat IETF as alpha3? (ditch the country part) + alpha3_map = {} + if config.ietf_as_alpha3: + for language in languages: + if language and language.country: + alpha3_map[language.alpha3] = language.country + language.country = None + + have_languages = video.subtitle_languages.copy() + if config.ietf_as_alpha3: + for language in have_languages: + if language and language.country: + alpha3_map[language.alpha3] = language.country + language.country = None + + missing_languages = (languages - have_languages) + + if config.any_language_is_enough != "Always search for all configured languages": + not_in_forced = "foreign" in config.any_language_is_enough + if "External or embedded subtitle" in config.any_language_is_enough: + langs = video.subtitle_languages if not not_in_forced else \ + filter(lambda l: not l.forced, video.subtitle_languages) + if langs: + Log.Debug("We have at least one subtitle for any configured language.") + return set() + + elif "External subtitle" in config.any_language_is_enough: + langs = video.external_subtitle_languages if not not_in_forced else \ + filter(lambda l: not l.forced, video.external_subtitle_languages) + if langs: + Log.Debug("We have at least one external subtitle for any configured language.") + return set() + + # all languages are found if we either really have subs for all languages or we only want to have exactly one language + # and we've only found one (the case for a selected language, Prefs['subtitles.only_one'] (one found sub matches any language)) + found_one_which_is_enough = len(video.subtitle_languages) >= 1 and Prefs['subtitles.only_one'] + if not missing_languages or found_one_which_is_enough: + if found_one_which_is_enough: + Log.Debug('Only one language was requested, and we\'ve got a subtitle for %s', video) + else: + Log.Debug('All languages %r exist for %s', languages, video) + return set() + + # re-add country codes to the missing languages, in case we've removed them above + if config.ietf_as_alpha3: + for language in languages: + language.country = alpha3_map.get(language.alpha3, None) + + return missing_languages + + +def pre_download_hook(subtitle): + if subtitle.is_pack: + # try retrieving the subtitle from a cached pack archive + pack_data = get_pack_data(subtitle) + if pack_data: + subtitle.pack_data = pack_data + + +def post_download_hook(subtitle): + # if a new pack was downloaded, store it in the cache; providers' download method is responsible for + # setting subtitle.pack_data to None in case the cached pack data we provided was successfully used + if subtitle.is_pack and subtitle.pack_data: + # store pack data in cache + store_pack_data(subtitle, subtitle.pack_data) + + # may be redundant + subtitle.pack_data = None + + +def language_hook(provider): + return config.get_lang_list(provider=provider) + + +def download_best_subtitles(video_part_map, min_score=0, throttle_time=None, providers=None): + hearing_impaired = Prefs['subtitles.search.hearingImpaired'] + languages = set([Language.rebuild(l) for l in config.lang_list]) + if not languages: + return + + use_videos = [] + missing_languages = set() + for video, part in video_part_map.iteritems(): + if not video.ignore_all: + p_missing_languages = get_missing_languages(video, part) + else: + p_missing_languages = languages + + if p_missing_languages: + Log.Info(u"%s has missing languages: %s", os.path.basename(video.name), p_missing_languages) + refine_video(video, refiner_settings=config.refiner_settings) + use_videos.append(video) + missing_languages.update(p_missing_languages) + + # prepare blacklist + blacklist = get_blacklist_from_part_map(video_part_map, languages) + + if use_videos and missing_languages: + Log.Debug("Download best subtitles using settings: min_score: %s, hearing_impaired: %s, languages: %s" % + (min_score, hearing_impaired, missing_languages)) + + return subliminal.download_best_subtitles(set(use_videos), missing_languages, min_score, hearing_impaired, + providers=providers or config.providers, + provider_configs=config.provider_settings, + pool_class=config.provider_pool, + compute_score=compute_score, throttle_time=throttle_time, + blacklist=blacklist, throttle_callback=config.provider_throttle, + pre_download_hook=pre_download_hook, + post_download_hook=post_download_hook, + language_hook=language_hook) + Log.Debug("All languages for all requested videos exist. Doing nothing.") \ No newline at end of file diff --git a/Contents/Code/support/extract.py b/Contents/Code/support/extract.py new file mode 100644 index 000000000..7e64b8643 --- /dev/null +++ b/Contents/Code/support/extract.py @@ -0,0 +1,214 @@ +# coding=utf-8 +import os +import subprocess +import traceback + +from support.helpers import quote_args, mswindows, get_title_for_video_metadata, cast_bool, \ + audio_streams_match_languages +from support.i18n import _ +from support.items import get_item_kind_from_item, refresh_item, get_all_items, get_item, MI_KEY +from support.storage import get_subtitle_storage, save_subtitles +from support.config import config +from support.history import get_history +from support.plex_media import get_all_parts, get_embedded_subtitle_streams, get_part, get_plex_metadata, \ + update_stream_info, is_stream_forced +from support.scanning import scan_videos +from subzero.language import Language +from subliminal_patch.subtitle import ModifiedSubtitle + + +def agent_extract_embedded(video_part_map, set_as_existing=False): + try: + subtitle_storage = get_subtitle_storage() + + to_extract = [] + item_count = 0 + + threads = [] + + for scanned_video, part_info in video_part_map.iteritems(): + plexapi_item = scanned_video.plexapi_metadata["item"] + stored_subs = subtitle_storage.load_or_new(plexapi_item) + valid_langs_in_media = \ + audio_streams_match_languages(scanned_video, config.get_lang_list(ordered=True)) + + if not config.lang_list.difference(valid_langs_in_media): + Log.Debug("Skipping embedded subtitle extraction for %s, audio streams are in correct language(s)", + plexapi_item.rating_key) + continue + + for plexapi_part in get_all_parts(plexapi_item): + item_count = item_count + 1 + used_one_unknown_stream = False + used_one_known_stream = False + for requested_language in config.lang_list: + skip_unknown = used_one_unknown_stream or used_one_known_stream + embedded_subs = stored_subs.get_by_provider(plexapi_part.id, requested_language, "embedded") + current = stored_subs.get_any(plexapi_part.id, requested_language) or \ + requested_language in scanned_video.external_subtitle_languages + + if not embedded_subs: + stream_data = get_embedded_subtitle_streams(plexapi_part, requested_language=requested_language, + skip_unknown=skip_unknown) + + if stream_data and stream_data[0]["language"]: + stream = stream_data[0]["stream"] + if stream_data[0]["is_unknown"]: + used_one_unknown_stream = True + else: + used_one_known_stream = True + + to_extract.append(({scanned_video: part_info}, plexapi_part, str(stream.index), + str(requested_language), not current)) + + if not cast_bool(Prefs["subtitles.search_after_autoextract"]) or set_as_existing: + scanned_video.subtitle_languages.update({requested_language}) + else: + Log.Debug("Skipping embedded subtitle extraction for %s, already got %r from %s", + plexapi_item.rating_key, requested_language, embedded_subs[0].id) + if to_extract: + Log.Info("Triggering extraction of %d embedded subtitles of %d items", len(to_extract), item_count) + threads.append(Thread.Create(multi_extract_embedded, stream_list=to_extract, refresh=True, with_mods=True, + single_thread=not config.advanced.auto_extract_multithread)) + return threads + except: + Log.Error("Something went wrong when auto-extracting subtitles, continuing: %s", traceback.format_exc()) + + +def multi_extract_embedded(stream_list, refresh=False, with_mods=False, single_thread=True, extract_mode="a", + history_storage=None): + def execute(): + for video_part_map, plexapi_part, stream_index, language, set_current in stream_list: + plexapi_item = video_part_map.keys()[0].plexapi_metadata["item"] + + extract_embedded_sub(rating_key=plexapi_item.rating_key, part_id=plexapi_part.id, + plex_item=plexapi_item, part=plexapi_part, scanned_videos=video_part_map, + stream_index=stream_index, set_current=set_current, + language=language, with_mods=with_mods, refresh=refresh, extract_mode=extract_mode, + history_storage=history_storage) + + if single_thread: + with Thread.Lock(key="extract_embedded"): + execute() + else: + execute() + + +def season_extract_embedded(rating_key, requested_language, with_mods=False, force=False, mode="season"): + # get stored subtitle info for item id + subtitle_storage = get_subtitle_storage() + + try: + rating_keys = [rating_key] + if mode == "series": + seasons = get_all_items(key="children", value=rating_key, base="library/metadata") + rating_keys = [season[MI_KEY] for season in seasons] + + for rating_key in rating_keys: + for data in get_all_items(key="children", value=rating_key, base="library/metadata"): + item = get_item(data[MI_KEY]) + if item: + stored_subs = subtitle_storage.load_or_new(item) + for part in get_all_parts(item): + embedded_subs = stored_subs.get_by_provider(part.id, requested_language, "embedded") + current = stored_subs.get_any(part.id, requested_language) + if not embedded_subs or force: + stream_data = get_embedded_subtitle_streams(part, requested_language=requested_language) + if stream_data: + stream = stream_data[0]["stream"] + + set_current = not current or force + refresh = not current + + extract_embedded_sub(rating_key=item.rating_key, part_id=part.id, + stream_index=str(stream.index), set_current=set_current, + refresh=refresh, language=requested_language, with_mods=with_mods, + extract_mode="m") + finally: + subtitle_storage.destroy() + + +def extract_embedded_sub(**kwargs): + rating_key = kwargs["rating_key"] + part_id = kwargs.pop("part_id") + stream_index = kwargs.pop("stream_index") + with_mods = kwargs.pop("with_mods", False) + language = Language.fromietf(kwargs.pop("language")) + refresh = kwargs.pop("refresh", True) + set_current = kwargs.pop("set_current", True) + + plex_item = kwargs.pop("plex_item", get_item(rating_key)) + item_type = get_item_kind_from_item(plex_item) + part = kwargs.pop("part", get_part(plex_item, part_id)) + scanned_videos = kwargs.pop("scanned_videos", None) + extract_mode = kwargs.pop("extract_mode", "a") + + any_successful = False + + from interface.menu_helpers import set_refresh_menu_state + + if part: + if not scanned_videos: + metadata = get_plex_metadata(rating_key, part_id, item_type, plex_item=plex_item) + scanned_videos = scan_videos([metadata], ignore_all=True, skip_hashing=True) + + update_stream_info(part) + for stream in part.streams: + # subtitle stream + if str(stream.index) == stream_index: + is_forced = is_stream_forced(stream) + bn = os.path.basename(part.file) + + set_refresh_menu_state(_(u"Extracting subtitle %(stream_index)s of %(filename)s", + stream_index=stream_index, + filename=bn)) + Log.Info(u"Extracting stream %s (%s) of %s", stream_index, str(language), bn) + + out_codec = stream.codec if stream.codec != "mov_text" else "srt" + + args = [ + config.plex_transcoder, "-i", part.file, "-map", "0:%s" % stream_index, "-f", out_codec, "-" + ] + + cmdline = quote_args(args) + Log.Debug(u"Calling: %s", cmdline) + if mswindows: + Log.Debug("MSWindows: Fixing encoding") + cmdline = cmdline.encode("mbcs") + + output = None + try: + output = subprocess.check_output(cmdline, stderr=subprocess.PIPE, shell=True) + except: + Log.Error("Extraction failed: %s", traceback.format_exc()) + + if output: + subtitle = ModifiedSubtitle(language, mods=config.default_mods if with_mods else None) + subtitle.content = output + subtitle.provider_name = "embedded" + subtitle.id = "stream_%s" % stream_index + subtitle.score = 0 + subtitle.set_encoding("utf-8") + + # fixme: speedup video; only video.name is needed + video = scanned_videos.keys()[0] + save_successful = save_subtitles(scanned_videos, {video: [subtitle]}, mode="m", + set_current=set_current) + set_refresh_menu_state(None) + + if save_successful and refresh: + refresh_item(rating_key) + + # add item to history + item_title = get_title_for_video_metadata(video.plexapi_metadata, + add_section_title=False, add_episode_title=True) + + history = get_history() + history.add(item_title, video.id, section_title=video.plexapi_metadata["section"], + thumb=video.plexapi_metadata["super_thumb"], + subtitle=subtitle, mode=extract_mode) + history.destroy() + + any_successful = True + + return any_successful \ No newline at end of file diff --git a/Contents/Code/support/helpers.py b/Contents/Code/support/helpers.py new file mode 100755 index 000000000..64a6d8fe8 --- /dev/null +++ b/Contents/Code/support/helpers.py @@ -0,0 +1,446 @@ +# coding=utf-8 +import os +import traceback +import types +import unicodedata +import datetime +import urllib +import time +import re +import platform +import subprocess +import sys +from collections import OrderedDict + +from babelfish.exceptions import LanguageError + +import chardet + +from bs4 import UnicodeDammit +from subzero.language import Language, language_from_stream +from subzero.analytics import track_event + +mswindows = (sys.platform == "win32") +if mswindows: + from subprocess import list2cmdline + quote_args = list2cmdline +else: + # POSIX + from pipes import quote + + def quote_args(seq): + return ' '.join(quote(arg) for arg in seq) + +# Unicode control characters can appear in ID3v2 tags but are not legal in XML. +RE_UNICODE_CONTROL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \ + u'|' + \ + u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \ + ( + unichr(0xd800), unichr(0xdbff), unichr(0xdc00), unichr(0xdfff), + unichr(0xd800), unichr(0xdbff), unichr(0xdc00), unichr(0xdfff), + unichr(0xd800), unichr(0xdbff), unichr(0xdc00), unichr(0xdfff) + ) + + +def cast_bool(value): + return str(value).strip() in ("true", "True") + + +def cast_int(value, default=None): + try: + return int(value) + except ValueError: + return default + + +# A platform independent way to split paths which might come in with different separators. +def split_path(str): + if str.find('\\') != -1: + return str.split('\\') + else: + return str.split('/') + + +def unicodize(s): + filename = s + try: + filename = unicodedata.normalize('NFC', unicode(s.decode('utf-8'))) + except: + Log('Failed to unicodize: ' + repr(filename)) + try: + filename = re.sub(RE_UNICODE_CONTROL, '', filename) + except: + Log('Couldn\'t strip control characters: ' + repr(filename)) + return filename + + +def force_unicode(s): + if not isinstance(s, types.UnicodeType): + try: + s = s.decode("utf-8") + except UnicodeDecodeError: + t = chardet.detect(s) + try: + s = s.decode(t["encoding"]) + except UnicodeDecodeError: + s = UnicodeDammit(s).unicode_markup + return s + + +def clean_filename(filename): + # this will remove any whitespace and punctuation chars and replace them with spaces, strip and return as lowercase + return string.translate(filename.encode('utf-8'), string.maketrans(string.punctuation + string.whitespace, + ' ' * len( + string.punctuation + string.whitespace))).strip().lower() + + +def is_recent(t): + now = datetime.datetime.now() + when = datetime.datetime.fromtimestamp(t) + value, key = Prefs["scheduler.item_is_recent_age"].split() + if now - datetime.timedelta(**{key: int(value)}) < when: + return True + return False + + +# thanks, Plex-Trakt-Scrobbler +def str_pad(s, length, align='left', pad_char=' ', trim=False): + if not s: + return s + + if not isinstance(s, (str, unicode)): + s = str(s) + + if len(s) == length: + return s + elif len(s) > length and not trim: + return s + + if align == 'left': + if len(s) > length: + return s[:length] + else: + return s + (pad_char * (length - len(s))) + elif align == 'right': + if len(s) > length: + return s[len(s) - length:] + else: + return (pad_char * (length - len(s))) + s + else: + raise ValueError("Unknown align type, expected either 'left' or 'right'") + + +def pad_title(value, width=49): + """Pad a title to 30 characters to force the 'details' view.""" + return str_pad(value, width, pad_char=' ') + + +def get_plex_item_display_title(item, kind, parent=None, parent_title=None, section_title=None, + add_section_title=False): + """ + :param item: plex item + :param kind: show or movie + :param parent: season or None + :param parent_title: parentTitle or None + :return: + """ + return get_video_display_title(kind, item.title, + section_title=( + section_title or (parent.section.title if parent and getattr(parent, "section") + else None)), + parent_title=(parent_title or (parent.show.title if parent else None)), + season=parent.index if parent else None, + episode=item.index if kind == "show" else None, + add_section_title=add_section_title) + + +def series_num(v): + try: + return int(v) + except (TypeError, ValueError): + pass + + +def get_video_display_title(kind, title, section_title=None, parent_title=None, season=None, episode=None, + add_section_title=False): + section_add = "" + if add_section_title: + section_add = ("%s: " % section_title) if section_title else "" + + if kind in ("season", "show") and parent_title: + if series_num(season) is not None and series_num(episode) is not None: + return '%s%s S%02dE%02d%s' % (section_add, parent_title, season or 0, episode or 0, + (", %s" % title if title else "")) + elif series_num(season) is not None: + return '%s%s S%02d%s' % (section_add, parent_title, season or 0, + (", %s" % title if title else "")) + + return '%s%s%s' % (section_add, parent_title, (", %s" % title if title else "")) + return "%s%s" % (section_add, title) + + +def get_title_for_video_metadata(metadata, add_section_title=True, add_episode_title=False): + """ + + :param metadata: + :param add_section_title: + :param add_episode_title: add the episode's title if its an episode else always add title + :return: + """ + # compute item title + add_title = (add_episode_title and metadata["series_id"]) or not metadata["series_id"] + return get_video_display_title( + "show" if metadata["series_id"] else "movie", + metadata["title"] if add_title else "", + parent_title=metadata.get("series", None), + season=metadata.get("season", None), + episode=metadata.get("episode", None), + section_title=metadata.get("section", None), + add_section_title=add_section_title + ) + + +def get_identifier(): + identifier = None + try: + identifier = Platform.MachineIdentifier + except: + pass + + if not identifier: + identifier = String.UUID() + + return Hash.SHA1(identifier + "SUBZEROOOOOOOOOO") + + +def encode_message(base, s): + return "%s?message=%s" % (base, urllib.quote_plus(s)) + + +def decode_message(s): + return urllib.unquote_plus(s) + + +def timestamp(): + return int(time.time()*1000) + + +def df(d): + return d.strftime("%Y-%m-%d %H:%M:%S") if d else "legacy data" + + +def query_plex(url, args): + """ + simple http query to the plex API without parsing anything too complicated + :param url: + :param args: + :return: + """ + use_args = args.copy() + + computed_args = "&".join(["%s=%s" % (key, String.Quote(value)) for key, value in use_args.iteritems()]) + + return HTTP.Request(url + ("?%s" % computed_args) if computed_args else "", immediate=True) + + +def check_write_permissions(path): + if platform.system() == "Windows": + # physical access check + check_path = os.path.join(os.path.realpath(path), ".sz_perm_chk") + try: + if os.path.exists(check_path): + os.rmdir(check_path) + os.mkdir(check_path) + os.rmdir(check_path) + return True + except OSError: + pass + + else: + # os.access check + return os.access(path, os.W_OK | os.X_OK) + return False + + +def get_item_hints(data): + """ + :param data: video item dict of media_to_videos + :return: + """ + hints = {"title": data["original_title"] or data["title"], "type": "movie"} + if data["type"] == "episode": + hints.update( + { + "type": "episode", + "episode_title": data["title"], + "title": data["original_title"] or data["series"], + } + ) + if hints["title"]: + hints["title"] = hints["title"].replace(":", "") + return hints + + +def notify_executable(exe_info, videos, subtitles, storage): + variables = ( + "subtitle_language", "subtitle_path", "subtitle_filename", "provider", "score", "storage", "series_id", + "series", "title", "section", "filename", "path", "folder", "season_id", "type", "id", "season" + ) + to_clean = ("PYTHONPATH", "PYTHONHOME") + exe, arguments = exe_info + for video, video_subtitles in subtitles.items(): + for subtitle in video_subtitles: + lang = str(subtitle.language) + data = video.plexapi_metadata.copy() + data.update({ + "subtitle_language": lang, + "provider": subtitle.provider_name, + "score": subtitle.score, + "storage": storage, + "subtitle_path": subtitle.storage_path, + "subtitle_filename": os.path.basename(subtitle.storage_path) + }) + + # fill missing data with None + prepared_data = dict((v, data.get(v)) for v in variables) + + prepared_arguments = [arg % prepared_data for arg in arguments] + + Log.Debug(u"Calling %s with arguments: %s" % (exe, prepared_arguments)) + if not mswindows: + env_path = {"PATH": os.pathsep.join( + [ + "/usr/local/bin", + "/usr/bin", + os.environ.get("PATH", "") + ] + ) + } + env = dict(os.environ, **env_path) + env.pop("LD_LIBRARY_PATH", None) + else: + env = dict(os.environ) + + # clean out any Plex-PYTHONPATH that may bleed through the spawned process + for v in to_clean: + if v in env and "plex" in env[v].lower(): + del env[v] + + try: + proc = subprocess.Popen(quote_args([exe] + prepared_arguments), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True, env=env, cwd=os.path.dirname(exe)) + output, errors = proc.communicate() + + if proc.returncode == 1: + Log.Error(u"Calling %s with args %s failed: output:\n%s, error:\n%s", exe, prepared_arguments, + output, errors) + return + + output = output.decode() + + except: + Log.Error(u"Calling %s failed: %s", exe, traceback.format_exc()) + else: + Log.Debug(u"Process output: %s", output) + + +def track_usage(category=None, action=None, label=None, value=None): + if not cast_bool(Prefs["track_usage"]): + return + + if "last_tracked" not in Dict: + Dict["last_tracked"] = OrderedDict() + Dict.Save() + + event_key = (category, action, label, value) + now = datetime.datetime.now() + if event_key in Dict["last_tracked"] and (Dict["last_tracked"][event_key] + datetime.timedelta(minutes=30)) < now: + return + + Dict["last_tracked"][event_key] = now + + # maintenance + for key, value in Dict["last_tracked"].copy().iteritems(): + # kill day old values + if value < now - datetime.timedelta(days=1): + try: + del Dict["last_tracked"][key] + except: + pass + + try: + Thread.Create(dispatch_track_usage, category, action, label, value, + identifier=Dict["anon_id"], first_use=Dict["first_use"], + add=Network.PublicAddress) + except: + Log.Debug("Something went wrong when reporting anonymous user statistics: %s", traceback.format_exc()) + + +def dispatch_track_usage(*args, **kwargs): + identifier = kwargs.pop("identifier") + first_use = kwargs.pop("first_use") + add = kwargs.pop("add") + try: + track_event(identifier=identifier, first_use=first_use, add=add, *[str(a) for a in args]) + except: + Log.Debug("Something went wrong when reporting anonymous user statistics: %s", traceback.format_exc()) + + +def get_language_from_stream(lang_code): + if lang_code: + lang = Locale.Language.Match(lang_code) + if lang and lang != "xx": + # Log.Debug("Found language: %r", lang) + return Language.fromietf(lang) + elif lang: + try: + return language_from_stream(lang_code) + except LanguageError: + pass + + +def audio_streams_match_languages(video, languages): + without_forced = filter(lambda x: not x.forced, languages) + if video.audio_languages and without_forced: + if Prefs["subtitles.when"] == "Always": + return set() + + elif Prefs["subtitles.when"] == "When main audio stream is not Subtitle Language (1)": + if video.audio_languages[0] == without_forced[0]: + return set(without_forced) + + elif Prefs["subtitles.when"] == "When any audio stream is not Subtitle Language (1)": + if without_forced[0] in video.audio_languages: + return set(without_forced) + + elif Prefs["subtitles.when"] == "When main audio stream is not any configured language": + if video.audio_languages[0] in without_forced: + return set(without_forced) + + elif Prefs["subtitles.when"] == "When any audio stream is not any configured language": + matching = set(video.audio_languages).intersection(set(without_forced)) + if matching: + return set(without_forced) + + # if Prefs["subtitles.when_forced"] in [ + # "Always", + # "Only for Subtitle Language (1)", + # "Only for Subtitle Language (2)", + # "Only for Subtitle Language (3)" + # ]: + + return set() + + +def get_language(lang_short): + return Language.fromietf(lang_short) + + +def display_language(l): + if not l: + return "Unknown" + return _(str(l.basename).lower()) + ((u" (%s)" % _("forced")) if l.forced else "") + + +class PartUnknownException(Exception): + pass diff --git a/Contents/Code/support/history.py b/Contents/Code/support/history.py new file mode 100644 index 000000000..890a08e8e --- /dev/null +++ b/Contents/Code/support/history.py @@ -0,0 +1,4 @@ +# coding=utf-8 +from subzero.history_storage import SubtitleHistory + +get_history = lambda: SubtitleHistory(Data, Thread, int(Prefs["history_size"])) diff --git a/Contents/Code/support/i18n.py b/Contents/Code/support/i18n.py new file mode 100644 index 000000000..f904d62f7 --- /dev/null +++ b/Contents/Code/support/i18n.py @@ -0,0 +1,109 @@ +# coding=utf-8 + +import inspect + +from support.config import config + + +core = getattr(Data, "_core") + + +# get original localization module in order to access its base classes later on +def get_localization_module(): + cls = getattr(core.localization, "__class__") + return inspect.getmodule(cls) + + +plex_i18n_module = get_localization_module() + + +def old_style_placeholders_count(s): + # fixme: incomplete, use regex + return sum(s.count(c) for c in ["%s", "%d", "%r", "%f", "%i"]) + + +def check_old_style_placeholders(k, args): + # replace escaped %'s? + k = k.__str__().replace("%%", "") + + if "%(" in k: + Log.Error(u"%r defines named placeholders for formatting" % k) + return "NEEDS NAMED ARGUMENTS" + + placeholders_found = old_style_placeholders_count(k) + if placeholders_found and not args: + Log.Error(u"%r requires a arguments for formatting" % k) + return "NEEDS FORMAT ARGUMENTS" + + elif not placeholders_found and args: + Log.Error(u"%r doesn't define placeholders for formatting" % k) + return "HAS NO FORMAT ARGUMENTS" + + elif placeholders_found and placeholders_found != len(args): + Log.Error(u"%r wrong amount of arguments supplied for formatting" % k) + return "WRONG FORMAT ARGUMENT COUNT" + + +class SmartLocalStringFormatter(plex_i18n_module.LocalStringFormatter): + """ + this allows the use of dictionaries for string formatting, also does some sanity checking on the keys and values + """ + def __init__(self, string1, string2, locale=None): + if isinstance(string2, tuple): + # dictionary passed + if len(string2) == 1 and hasattr(string2[0], "iteritems"): + string2 = string2[0] + if config.debug_i18n: + if "%(" not in string1.__str__().replace("%%", ""): + Log.Error(u"%r: dictionary for non-named format string supplied" % string1.__str__()) + string1 = "%s" + string2 = "NO NAMED ARGUMENTS" + + # arguments + elif len(string2) >= 1 and config.debug_i18n: + msg = check_old_style_placeholders(string1, string2) + if msg: + string1 = "%s" + string2 = msg + + setattr(self, "_string1", string1) + setattr(self, "_string2", string2) + setattr(self, "_locale", locale) + + +def local_string_with_optional_format(key, *args, **kwargs): + if kwargs: + args = (kwargs,) + else: + args = tuple(args) + + if args: + # fixme: may not be the best idea as this evaluates the string early + try: + return unicode(SmartLocalStringFormatter(plex_i18n_module.LocalString(core, key, Locale.CurrentLocale), args)) + except (TypeError, ValueError): + Log.Exception("Broken translation!") + Log.Debug("EN string: %s", plex_i18n_module.LocalString(core, key, "en")) + Log.Debug("%s string: %r", Locale.CurrentLocale, + unicode(plex_i18n_module.LocalString(core, key, Locale.CurrentLocale))) + return unicode(SmartLocalStringFormatter(plex_i18n_module.LocalString(core, key, "en"), args)) + + # check string instances for arguments + if config.debug_i18n: + msg = check_old_style_placeholders(key, args) + if msg: + return msg + + try: + return unicode(plex_i18n_module.LocalString(core, key, Locale.CurrentLocale)) + + except TypeError: + Log.Exception("Broken translation!") + return unicode(plex_i18n_module.LocalString(core, key, "en")) + + +_ = local_string_with_optional_format + + +def is_localized_string(s): + return hasattr(s, "localize") diff --git a/Contents/Code/support/ignore.py b/Contents/Code/support/ignore.py new file mode 100644 index 000000000..6ddd94201 --- /dev/null +++ b/Contents/Code/support/ignore.py @@ -0,0 +1,75 @@ +# coding=utf-8 + +from subzero.lib.dict import DictProxy +from config import config + + +class ExcludeDict(DictProxy): + store = "ignore" + + # single item keys returned by helpers.items.getItems mapped to their parents + translate_keys = { + "section": "sections", + "show": "series", + "movie": "videos", + "episode": "videos", + "season": "seasons", + } + + # getItems types mapped to their verbose names + keys_verbose = { + "sections": "Section", + "series": "Series", + "videos": "Item", + "seasons": "Season", + } + + key_order = ("sections", "series", "videos", "seasons") + + def __len__(self): + try: + return sum(len(self.Dict[self.store][key]) for key in self.key_order) + except KeyError: + # old version + self.Dict[self.store] = self.setup_defaults() + return 0 + + def translate_key(self, name): + return self.translate_keys.get(name) + + def verbose(self, name): + return self.keys_verbose.get(self.translate_key(name) or name) + + def get_title_key(self, kind, key): + return "%s_%s" % (kind, key) + + def add_title(self, kind, key, title): + self["titles"][self.get_title_key(kind, key)] = title + + def remove_title(self, kind, key): + title_key = self.get_title_key(kind, key) + if title_key in self.titles: + del self.titles[title_key] + + def get_title(self, kind, key): + title_key = self.get_title_key(kind, key) + if title_key in self.titles: + return self.titles[title_key] + + def save(self): + Dict.Save() + + def setup_defaults(self): + return {"sections": [], "series": [], "videos": [], "titles": {}, "seasons": []} + + +class IncludeDict(ExcludeDict): + store = "include" + + +exclude_list = ExcludeDict(Dict) +include_list = IncludeDict(Dict) + + +def get_decision_list(): + return include_list if config.include else exclude_list diff --git a/Contents/Code/support/items.py b/Contents/Code/support/items.py new file mode 100644 index 000000000..c51df76b0 --- /dev/null +++ b/Contents/Code/support/items.py @@ -0,0 +1,481 @@ +# coding=utf-8 + +import logging +import re +import traceback +import types +import os + +import time + +import datetime + +from ignore import get_decision_list +from helpers import is_recent, get_plex_item_display_title, query_plex, PartUnknownException +from lib import Plex, get_intent +from config import config +from subliminal_patch.subtitle import ModifiedSubtitle +from subzero.modification import registry as mod_registry, SubtitleModifications +from socket import timeout + +logger = logging.getLogger(__name__) + +MI_KIND, MI_TITLE, MI_KEY, MI_DEEPER, MI_ITEM = 0, 1, 2, 3, 4 + +container_size_re = re.compile(ur'totalSize="(\d+)"') + + +def get_item(key): + try: + item_id = int(key) + except ValueError: + return + + try: + item_container = Plex["library"].metadata(item_id) + except timeout: + Log.Debug("PMS API timed out when querying information about item %d", item_id) + return + + try: + return list(item_container)[0] + except: + pass + + +def get_item_kind(item): + return type(item).__name__ + + +PLEX_API_TYPE_MAP = { + "Show": "series", + "Season": "season", + "Episode": "episode", + "Movie": "movie", +} + + +def get_item_kind_from_rating_key(key): + item = get_item(key) + return PLEX_API_TYPE_MAP.get(get_item_kind(item)) + + +def get_item_kind_from_item(item): + return PLEX_API_TYPE_MAP.get(get_item_kind(item)) + + +def get_item_title(item): + kind = get_item_kind_from_item(item) + if kind not in ("episode", "movie", "season", "series"): + return + + if kind == "episode": + return get_plex_item_display_title(item, "show", parent=item.season, section_title=None, + parent_title=item.show.title) + elif kind == "season": + return get_plex_item_display_title(item, "season", parent=item.show, section_title="Season", + parent_title=item.show.title) + else: + return get_plex_item_display_title(item, kind, section_title=None) + + +def get_item_thumb(item): + kind = get_item_kind(item) + if kind == "Episode": + return item.show.thumb + elif kind == "Section": + return item.art + return item.thumb + + +def get_items_info(items): + return items[0][MI_KIND], items[0][MI_DEEPER] + + +def get_kind(items): + return items[0][MI_KIND] + + +def get_section_size(key): + """ + quick query to determine the section size + :param key: + :return: + """ + size = None + url = "http://127.0.0.1:32400/library/sections/%s/all" % int(key) + use_args = { + "X-Plex-Container-Size": "0", + "X-Plex-Container-Start": "0" + } + response = query_plex(url, use_args) + matches = container_size_re.findall(response.content) + if matches: + size = int(matches[0]) + + return size + + +def get_items(key="recently_added", base="library", value=None, flat=False, add_section_title=False): + """ + try to handle all return types plex throws at us and return a generalized item tuple + """ + items = [] + apply_value = None + if value: + if isinstance(value, types.ListType): + apply_value = value + else: + apply_value = [value] + result = getattr(Plex[base], key)(*(apply_value or [])) + + for item in result: + cls = getattr(getattr(item, "__class__"), "__name__") + if hasattr(item, "scanner"): + kind = "section" + elif cls == "Directory": + kind = "directory" + else: + kind = item.type + + # only return items for our enabled sections + section_key = None + if kind == "section": + section_key = item.key + else: + if hasattr(item, "section_key"): + section_key = getattr(item, "section_key") + + if section_key and section_key not in config.enabled_sections: + continue + + if kind == "season": + # fixme: i think this case is unused now + if flat: + # return episodes + for child in item.children(): + items.append(("episode", get_plex_item_display_title(child, "show", parent=item, add_section_title=add_section_title), int(item.rating_key), + False, child)) + else: + # return seasons + items.append(("season", item.title, int(item.rating_key), True, item)) + + elif kind == "directory": + items.append(("directory", item.title, item.key, True, item)) + + elif kind == "section": + if item.type in ['movie', 'show']: + item.size = get_section_size(item.key) + items.append(("section", item.title, int(item.key), True, item)) + + elif kind == "episode": + items.append( + (kind, get_plex_item_display_title(item, "show", parent=item.season, parent_title=item.show.title, section_title=item.section.title, + add_section_title=add_section_title), int(item.rating_key), False, item)) + + elif kind in ("movie", "artist", "photo"): + items.append((kind, get_plex_item_display_title(item, kind, section_title=item.section.title, add_section_title=add_section_title), + int(item.rating_key), False, item)) + + elif kind == "show": + items.append(( + kind, get_plex_item_display_title(item, kind, section_title=item.section.title, add_section_title=add_section_title), int(item.rating_key), True, + item)) + + return items + + +def get_recent_items(): + """ + actually get the recent items, not limited like /library/recentlyAdded + :return: + """ + args = { + "sort": "addedAt:desc", + "X-Plex-Container-Start": "0", + "X-Plex-Container-Size": "%s" % config.max_recent_items_per_library + } + + episode_re = re.compile(ur'(?su)ratingKey="(?P\d+)"' + ur'.+?grandparentRatingKey="(?P\d+)"' + ur'.+?title="(?P.*?)"' + ur'.+?grandparentTitle="(?P<parent_title>.*?)"' + ur'.+?index="(?P<episode>\d+?)"' + ur'.+?parentIndex="(?P<season>\d+?)".+?addedAt="(?P<added>\d+)"' + ur'.+?<Part.+? file="(?P<filename>[^"]+?)"') + movie_re = re.compile(ur'(?su)ratingKey="(?P<key>\d+)".+?title="(?P<title>.*?)' + ur'".+?addedAt="(?P<added>\d+)"' + ur'.+?<Part.+? file="(?P<filename>[^"]+?)"') + available_keys = ("key", "title", "parent_key", "parent_title", "season", "episode", "added", "filename") + recent = [] + + ref_list = get_decision_list() + + for section in Plex["library"].sections(): + if section.type not in ("movie", "show") \ + or section.key not in config.enabled_sections \ + or ((section.key not in ref_list.sections and config.include) + or (section.key in ref_list.sections and not config.include)): + Log.Debug(u"Skipping section: %s" % section.title) + continue + + use_args = args.copy() + plex_item_type = "Movie" + if section.type == "show": + use_args["type"] = "4" + plex_item_type = "Episode" + + url = "http://127.0.0.1:32400/library/sections/%s/all" % int(section.key) + response = query_plex(url, use_args) + + matcher = episode_re if section.type == "show" else movie_re + matches = [m.groupdict() for m in matcher.finditer(response.content)] + for match in matches: + data = dict((key, match[key] if key in match else None) for key in available_keys) + if section.type == "show" and ((data["parent_key"] not in ref_list.series and config.include) or + (data["parent_key"] in ref_list.series and not config.include)): + Log.Debug(u"Skipping series: %s" % data["parent_title"]) + continue + if (data["key"] not in ref_list.videos and config.include) or \ + (data["key"] in ref_list.videos and not config.include): + Log.Debug(u"Skipping item: %s" % data["title"]) + continue + if not is_physically_wanted(data["filename"], plex_item_type): + Log.Debug(u"Skipping item (physically not wanted): %s" % data["title"]) + continue + + if is_recent(int(data["added"])): + recent.append((int(data["added"]), section.type, section.title, data["key"])) + + return recent + + +def get_on_deck_items(): + return get_items(key="on_deck", add_section_title=True) + + +def get_recently_added_items(): + return get_items(key="recently_added", add_section_title=True, flat=False) + + +def get_all_items(key, base="library", value=None, flat=False): + return get_items(key, base=base, value=value, flat=flat) + + +def is_wanted(rating_key, item=None): + """ + check whether an item, its show/season/section is in the soft or the hard ignore list + :param rating_key: + :param item: + :return: + """ + + ref_list = get_decision_list() + ret_val = ref_list.store == "include" + inc_exc_verbose = "exclude" if not ret_val else "include" + + # item in soft include/exclude list + if ref_list["videos"] and rating_key in ref_list["videos"]: + Log.Debug("Item %s is in the soft %s list" % (rating_key, inc_exc_verbose)) + return ret_val + + item = item or get_item(rating_key) + kind = get_item_kind(item) + + # show in soft include/exclude list + if kind == "Episode" and ref_list["series"] and item.show.rating_key in ref_list["series"]: + Log.Debug("Item %s's show is in the soft %s list" % (rating_key, inc_exc_verbose)) + return ret_val + + # season in soft include/exclude list + if kind == "Episode" and ref_list["seasons"] and item.season.rating_key in ref_list["seasons"]: + Log.Debug("Item %s's season is in the soft %s list" % (rating_key, inc_exc_verbose)) + return ret_val + + # section in soft include/exclude list + if ref_list["sections"] and item.section.key in ref_list["sections"]: + Log.Debug("Item %s's section is in the soft %s list" % (rating_key, inc_exc_verbose)) + return ret_val + + # physical/path include/exclude + if config.include_exclude_sz_files or config.include_exclude_paths: + for media in item.media: + for part in media.parts: + return is_physically_wanted(part.file, kind) + + return not ret_val + + +def is_physically_wanted(fn, kind): + if config.include_exclude_sz_files or config.include_exclude_paths: + # normally check current item folder and the library + check_paths = [".", "../"] + if kind == "Episode": + # series/episode, we've got a season folder here, also + check_paths.append("../../") + + wanted_results = [] + if config.include_exclude_sz_files: + for sub_path in check_paths: + wanted_results.append(config.is_physically_wanted( + os.path.normpath(os.path.join(os.path.dirname(fn), sub_path)), fn)) + + if config.include_exclude_paths: + wanted_results.append(config.is_path_wanted(fn)) + + if config.include and any(wanted_results): + return True + elif not config.include and not all(wanted_results): + return False + + return not config.include + + +def refresh_item(rating_key, force=False, timeout=8000, refresh_kind=None, parent_rating_key=None): + intent = get_intent() + + # timeout actually is the time for which the intent will be valid + if force: + Log.Debug("Setting intent for force-refresh of %s to timeout: %s", rating_key, timeout) + intent.set("force", rating_key, timeout=timeout) + + # force Dict.Save() + intent.store.save() + + refresh = [rating_key] + + if refresh_kind == "season": + # season refresh, needs explicit per-episode refresh + refresh = [item.rating_key for item in list(Plex["library/metadata"].children(int(rating_key)))] + + multiple = len(refresh) > 1 + wait = config.advanced.get("refresh_after_called", 5) + Thread.Sleep(wait) + for key in refresh: + Log.Info("%s item %s", "Refreshing" if not force else "Forced-refreshing", key) + Plex["library/metadata"].refresh(key) + if multiple: + Thread.Sleep(wait) + + +def get_current_sub(rating_key, part_id, language, plex_item=None): + from support.storage import get_subtitle_storage + + item = plex_item or get_item(rating_key) + subtitle_storage = get_subtitle_storage() + stored_subs = subtitle_storage.load_or_new(item) + current_sub = stored_subs.get_any(part_id, language) + return current_sub, stored_subs, subtitle_storage + + +def save_stored_sub(stored_subtitle, rating_key, part_id, language, item_type, plex_item=None, storage=None, + stored_subs=None): + """ + in order for this to work, if the calling supplies stored_subs and storage, it has to trigger its saving and + destruction explicitly + :param stored_subtitle: + :param rating_key: + :param part_id: + :param language: + :param item_type: + :param plex_item: + :param storage: + :param stored_subs: + :return: + """ + from support.plex_media import get_plex_metadata + from support.scanning import scan_videos + from support.storage import save_subtitles, get_subtitle_storage + + plex_item = plex_item or get_item(rating_key) + + stored_subs_was_provided = True + if not stored_subs or not storage: + storage = get_subtitle_storage() + stored_subs = storage.load(plex_item.rating_key) + stored_subs_was_provided = False + + if not all([plex_item, stored_subs]): + return + + try: + metadata = get_plex_metadata(rating_key, part_id, item_type, plex_item=plex_item) + except PartUnknownException: + return + + scanned_parts = scan_videos([metadata], ignore_all=True, skip_hashing=True) + video, plex_part = scanned_parts.items()[0] + + subtitle = ModifiedSubtitle(language, mods=stored_subtitle.mods) + subtitle.content = stored_subtitle.content + if stored_subtitle.encoding: + # thanks plex + setattr(subtitle, "_guessed_encoding", stored_subtitle.encoding) + + if stored_subtitle.encoding != "utf-8": + subtitle.normalize() + stored_subtitle.content = subtitle.content + stored_subtitle.encoding = "utf-8" + storage.save(stored_subs) + + subtitle.plex_media_fps = plex_part.fps + subtitle.page_link = stored_subtitle.id + subtitle.language = language + subtitle.id = stored_subtitle.id + + try: + save_subtitles(scanned_parts, {video: [subtitle]}, mode="m", bare_save=True) + stored_subtitle.mods = subtitle.mods + Log.Debug("Modified %s subtitle for: %s:%s with: %s", language.name, rating_key, part_id, + ", ".join(subtitle.mods) if subtitle.mods else "none") + except: + Log.Error("Something went wrong when modifying subtitle: %s", traceback.format_exc()) + + if subtitle.storage_path: + stored_subtitle.last_mod = datetime.datetime.fromtimestamp(os.path.getmtime(subtitle.storage_path)) + + if not stored_subs_was_provided: + storage.save(stored_subs) + storage.destroy() + + +def set_mods_for_part(rating_key, part_id, language, item_type, mods, mode="add"): + plex_item = get_item(rating_key) + + if not plex_item: + return + + current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language, plex_item=plex_item) + if mode == "add": + for mod in mods: + identifier, args = SubtitleModifications.parse_identifier(mod) + mod_class = SubtitleModifications.get_mod_class(identifier) + + if identifier not in mod_registry.mods_available: + raise NotImplementedError("Mod unknown or not registered") + + # clean exclusive mods + if mod_class.exclusive and current_sub.mods: + for current_mod in current_sub.mods[:]: + if current_mod.startswith(identifier): + current_sub.mods.remove(current_mod) + Log.Info("Removing superseded mod %s" % current_mod) + + current_sub.add_mod(mod) + elif mode == "clear": + current_sub.add_mod(None) + elif mode == "remove": + for mod in mods: + current_sub.mods.remove(mod) + + elif mode == "remove_last": + if current_sub.mods: + current_sub.mods.pop() + else: + raise NotImplementedError("Wrong mode given") + + save_stored_sub(current_sub, rating_key, part_id, language, item_type, plex_item=plex_item, storage=storage, + stored_subs=stored_subs) + + storage.save(stored_subs) + storage.destroy() diff --git a/Contents/Code/support/lib.py b/Contents/Code/support/lib.py new file mode 100644 index 000000000..0acf65231 --- /dev/null +++ b/Contents/Code/support/lib.py @@ -0,0 +1,56 @@ +# coding=utf-8 + +import plex +from subzero.intent import TempIntent +from subzero.lib.dict import DictProxy +from subzero.lib.httpfake import PlexPyNativeResponseProxy +from subzero.constants import DEFAULT_TIMEOUT + + +class PlexPyNativeRequestProxy(object): + """ + A really dumb object that tries to mimic requests.Request in an incomplete way, so that plex.Plex + uses native plex HTTPRequests instead of the better requests.Request class. + + This allows us to operate freely on 127.0.0.1's PMS. + + To be used in conjunction with subzero.lib.httpfake.PlexPyNativeResponseProxy + """ + url = None + data = None + headers = None + method = None + + def prepare(self): + return self + + def send(self): + # fixme: add self.data to HTTP.Request + data = None + status_code = 200 + try: + data = HTTP.Request(self.url, headers=self.headers, immediate=True, method=self.method, + timeout=DEFAULT_TIMEOUT) + except Ex.HTTPError as e: + status_code = e.code + return PlexPyNativeResponseProxy(data, status_code, self) + + +plex.request.Request = PlexPyNativeRequestProxy + +Plex = plex.Plex + + +class IntentDictStorage(DictProxy): + store = "intent" + + def setup_defaults(self): + return {"force": {}} + + +def get_intent(): + """ + use this to get an intent from inside a separate thread + :return: + """ + return TempIntent(store=IntentDictStorage(Dict)) diff --git a/Contents/Code/support/localmedia.py b/Contents/Code/support/localmedia.py new file mode 100755 index 000000000..0c7d8b318 --- /dev/null +++ b/Contents/Code/support/localmedia.py @@ -0,0 +1,208 @@ +# coding=utf-8 + +import os + +import config +import helpers +import subtitlehelpers + +from config import config as sz_config +from subzero.language import ENDSWITH_LANGUAGECODE_RE + + +SECONDARY_TAGS = ['forced', 'normal', 'default', 'embedded', 'embedded-forced', 'custom', 'hi', 'cc', 'sdh'] + + +def find_subtitles(part, ignore_parts_cleanup=None): + lang_sub_map = {} + ignore_parts_cleanup = ignore_parts_cleanup or [] + part_filename = helpers.unicodize(part.file) + part_basename = os.path.splitext(os.path.basename(part_filename))[0] + use_filesystem = helpers.cast_bool(Prefs["subtitles.save.filesystem"]) + sub_dir_custom = Prefs["subtitles.save.subFolder.Custom"].strip() \ + if Prefs["subtitles.save.subFolder.Custom"] else None + + use_sub_subfolder = Prefs["subtitles.save.subFolder"] != "current folder" and not sub_dir_custom + autoclean = helpers.cast_bool(Prefs["subtitles.autoclean"]) + sub_subfolder = None + paths = [os.path.dirname(part_filename)] if use_filesystem else [] + + global_folders = [] + + if use_filesystem: + # Check for local subtitles subdirectory + sub_dir_base = paths[0] + sub_dir_list = [] + + if use_sub_subfolder: + # got selected subfolder + sub_subfolder = os.path.join(sub_dir_base, Prefs["subtitles.save.subFolder"]) + sub_dir_list.append(sub_subfolder) + sub_subfolder = os.path.normpath(helpers.unicodize(sub_subfolder)) + + if sub_dir_custom: + # got custom subfolder + sub_dir_custom = os.path.normpath(sub_dir_custom) + if os.path.isdir(sub_dir_custom) and os.path.isabs(sub_dir_custom): + # absolute folder + sub_dir_list.append(sub_dir_custom) + global_folders.append(sub_dir_custom) + else: + # relative folder + fld = os.path.join(sub_dir_base, sub_dir_custom) + sub_dir_list.append(fld) + + for sub_dir in sub_dir_list: + if os.path.isdir(sub_dir): + paths.append(sub_dir) + + # Check for a global subtitle location + global_subtitle_folder = os.path.join(Core.app_support_path, 'Subtitles') + if os.path.exists(global_subtitle_folder): + paths.append(global_subtitle_folder) + global_folders.append(global_subtitle_folder) + + # normalize all paths + paths = [os.path.normpath(helpers.unicodize(path)) for path in paths] + + # We start by building a dictionary of files to their absolute paths. We also need to know + # the number of media files that are actually present, in case the found local media asset + # is limited to a single instance per media file. + # + file_paths = {} + total_media_files = 0 + media_files = [] + for path in paths: + for file_path_listing in os.listdir(path.encode(sz_config.fs_encoding)): + # When using os.listdir with a unicode path, it will always return a string using the + # NFD form. However, we internally are using the form NFC and therefore need to convert + # it to allow correct regex / comparisons to be performed. + # + file_path_listing = helpers.unicodize(file_path_listing) + if os.path.isfile(os.path.join(path, file_path_listing).encode(sz_config.fs_encoding)): + file_paths[file_path_listing.lower()] = os.path.join(path, file_path_listing) + + # If we've found an actual media file, we should record it. + (root, ext) = os.path.splitext(file_path_listing) + if ext.lower()[1:] in config.VIDEO_EXTS: + total_media_files += 1 + + # collect found media files + media_files.append(root) + + # cleanup any leftover subtitle if no associated media file was found + if autoclean and ignore_parts_cleanup: + Log.Info("Skipping housekeeping of: %s", paths) + + if use_filesystem and autoclean and not ignore_parts_cleanup: + for path in paths: + # only housekeep in sub_subfolder if sub_subfolder is used + if use_sub_subfolder and path != sub_subfolder and not sz_config.advanced.thorough_cleaning: + continue + + # we can't housekeep the global subtitle folders as we don't know about *all* media files + # in a library; skip them + skip_path = False + for fld in global_folders: + if path.startswith(fld): + Log.Info("Skipping housekeeping of folder: %s", path) + skip_path = True + break + + if skip_path: + continue + + for file_path_listing in os.listdir(path.encode(sz_config.fs_encoding)): + file_path_listing = helpers.unicodize(file_path_listing) + enc_fn = os.path.join(path, file_path_listing).encode(sz_config.fs_encoding) + + if os.path.isfile(enc_fn): + (root, ext) = os.path.splitext(file_path_listing) + # it's a subtitle file + if ext.lower()[1:] in config.SUBTITLE_EXTS_BASE: + # get fn without forced/default/normal tag + split_tag = root.rsplit(".", 1) + if len(split_tag) > 1 and split_tag[1].lower() in SECONDARY_TAGS: + root = split_tag[0] + + # get associated media file name without language + sub_fn = ENDSWITH_LANGUAGECODE_RE.sub("", root) + + # subtitle basename and basename without possible language tag not found in collected + # media files? kill. + if root not in media_files and sub_fn not in media_files: + Log.Info("Removing leftover subtitle: %s", os.path.join(path, file_path_listing)) + try: + os.remove(enc_fn) + except (OSError, IOError): + Log.Error("Removing failed") + + Log('Looking for subtitle media in %d paths with %d media files.', len(paths), total_media_files) + Log('Paths: %s', ", ".join([helpers.unicodize(p) for p in paths])) + + for file_path in file_paths.values(): + local_filename = os.path.basename(file_path) + bn, ext = os.path.splitext(local_filename) + local_basename = helpers.unicodize(bn) + + # get fn without forced/default/normal tag + split_tag = local_basename.rsplit(".", 1) + has_additional_tag = False + if len(split_tag) > 1 and split_tag[1].lower() in SECONDARY_TAGS: + local_basename = split_tag[0] + has_additional_tag = True + + # split off possible language tag + local_basename2 = local_basename.rsplit('.', 1)[0] + filename_matches_part = local_basename == part_basename or local_basename2 == part_basename + filename_contains_part = part_basename in local_basename + + if not ext.lower()[1:] in config.SUBTITLE_EXTS: + continue + + # if the file is located within the global subtitle folders and its name doesn't match exactly, ignore it + if global_folders and not filename_matches_part: + skip_path = False + for fld in global_folders: + if file_path.startswith(fld): + skip_path = True + break + + if skip_path: + continue + + # determine whether to pick up the subtitle based on our match strictness + if not filename_matches_part: + if sz_config.ext_match_strictness == "strict" or ( + sz_config.ext_match_strictness == "loose" and not filename_contains_part): + # Log.Debug("%s doesn't match %s, skipping" % (helpers.unicodize(local_filename), + # helpers.unicodize(part_basename))) + continue + + subtitle_helper = subtitlehelpers.subtitle_helpers(file_path) + if subtitle_helper is not None: + local_lang_map = subtitle_helper.process_subtitles(part) + for new_language, subtitles in local_lang_map.items(): + + # Add the possible new language along with the located subtitles so that we can validate them + # at the end... + # + if not lang_sub_map.has_key(new_language): + lang_sub_map[new_language] = [] + lang_sub_map[new_language] = lang_sub_map[new_language] + subtitles + + # add known metadata subs to our sub list + if not use_filesystem: + for language, sub_list in subtitlehelpers.get_subtitles_from_metadata(part).iteritems(): + if sub_list: + if language not in lang_sub_map: + lang_sub_map[language] = [] + lang_sub_map[language] = lang_sub_map[language] + sub_list + + # Now whack subtitles that don't exist anymore. + for language in lang_sub_map.keys(): + part.subtitles[language].validate_keys(lang_sub_map[language]) + + # Now whack the languages that don't exist anymore. + for language in list(set(part.subtitles.keys()) - set(lang_sub_map.keys())): + part.subtitles[language].validate_keys({}) diff --git a/Contents/Code/support/missing_subtitles.py b/Contents/Code/support/missing_subtitles.py new file mode 100755 index 000000000..1f3b41498 --- /dev/null +++ b/Contents/Code/support/missing_subtitles.py @@ -0,0 +1,195 @@ +# coding=utf-8 +import traceback +import time + +import os + +from babelfish import LanguageReverseError + +from support.config import config, TEXT_SUBTITLE_EXTS +from support.helpers import get_plex_item_display_title, cast_bool, get_language_from_stream +from support.plex_media import is_stream_forced, update_stream_info +from support.items import get_item +from support.lib import Plex +from support.storage import get_subtitle_storage +from subzero.video import has_external_subtitle +from subzero.language import Language + + +def item_discover_missing_subs(rating_key, kind="show", added_at=None, section_title=None, internal=False, external=True, languages=()): + item_id = int(rating_key) + item = get_item(rating_key) + + if kind == "show": + item_title = get_plex_item_display_title(item, kind, parent=item.season, section_title=section_title, parent_title=item.show.title) + else: + item_title = get_plex_item_display_title(item, kind, section_title=section_title) + + subtitle_storage = get_subtitle_storage() + stored_subs = subtitle_storage.load(rating_key) + subtitle_storage.destroy() + + subtitle_target_dir, tdir_is_absolute = config.subtitle_sub_dir + + missing = set() + languages_set = set([Language.rebuild(l) for l in languages]) + for media in item.media: + existing_subs = {"internal": [], "external": [], "own_external": [], "count": 0} + for part in media.parts: + update_stream_info(part) + # did we already download an external subtitle before? + if subtitle_target_dir and stored_subs: + for language in languages_set: + if has_external_subtitle(part.id, stored_subs, language): + # check the existence of the actual subtitle file + + # get media filename without extension + part_basename = os.path.splitext(os.path.basename(part.file))[0] + + # compute target directory for subtitle + # fixme: move to central location + if tdir_is_absolute: + possible_subtitle_path_base = subtitle_target_dir + else: + possible_subtitle_path_base = os.path.join(os.path.dirname(part.file), subtitle_target_dir) + + possible_subtitle_path_base = os.path.realpath(possible_subtitle_path_base) + + # folder actually exists? + if not os.path.isdir(possible_subtitle_path_base): + continue + + found_any = False + for ext in config.subtitle_formats: + if cast_bool(Prefs['subtitles.only_one']): + possible_subtitle_path = os.path.join(possible_subtitle_path_base, + u"%s.%s" % (part_basename, ext)) + else: + possible_subtitle_path = os.path.join(possible_subtitle_path_base, + u"%s.%s.%s" % (part_basename, language, ext)) + + # check for subtitle existence + if os.path.isfile(possible_subtitle_path): + found_any = True + Log.Debug(u"Found: %s", possible_subtitle_path) + break + + if found_any: + existing_subs["own_external"].append(language) + existing_subs["count"] = existing_subs["count"] + 1 + + for stream in part.streams: + if stream.stream_type == 3: + is_forced = is_stream_forced(stream) + if stream.index: + key = "internal" + else: + key = "external" + + if not config.exotic_ext and stream.codec.lower() not in TEXT_SUBTITLE_EXTS: + continue + + # treat unknown language as lang1? + if not stream.language_code and config.treat_und_as_first: + lang = Language.rebuild(list(config.lang_list)[0]) + + # we can't parse empty language codes + elif not stream.language_code or not stream.codec: + continue + + else: + # parse with internal language parser first + try: + lang = get_language_from_stream(stream.language_code) + if not lang: + if config.treat_und_as_first: + lang = Language.rebuild(list(config.lang_list)[0]) + else: + continue + + except (ValueError, LanguageReverseError): + continue + + if lang: + # Log.Debug("Found babelfish language: %r", lang) + lang.forced = is_forced + existing_subs[key].append(lang) + existing_subs["count"] = existing_subs["count"] + 1 + + missing_from_part = set([Language.rebuild(l) for l in languages]) + if existing_subs["count"]: + + # fixme: this is actually somewhat broken with IETF, as Plex doesn't store the country portion + # (pt instead of pt-BR) inside the database. So it might actually download pt-BR if there's a local pt-BR + # subtitle but not our own. + existing_flat = set((existing_subs["internal"] if internal else []) + + (existing_subs["external"] if external else []) + + existing_subs["own_external"]) + + check_languages = set([Language.rebuild(l) for l in languages]) + alpha3_map = {} + if config.ietf_as_alpha3: + for language in existing_flat: + if language.country: + alpha3_map[language.alpha3] = language.country + language.country = None + + for language in check_languages: + if language.country: + alpha3_map[language.alpha3] = language.country + language.country = None + + # compare sets of strings, not sets of different Language instances + check_languages_str = set(str(l) for l in check_languages) + existing_flat_str = set(str(l) for l in existing_flat) + + if check_languages_str.issubset(existing_flat_str) or \ + (len(existing_flat) >= 1 and Prefs['subtitles.only_one']): + # all subs found + #Log.Info(u"All subtitles exist for '%s'", item_title) + continue + + missing_from_part = set(Language.fromietf(l) for l in check_languages_str - existing_flat_str) + if config.ietf_as_alpha3: + for language in missing_from_part: + language.country = alpha3_map.get(language.alpha3, None) + + if missing_from_part: + Log.Info(u"Subs still missing for '%s' (%s: %s): %s", item_title, rating_key, media.id, + missing_from_part) + missing.update(missing_from_part) + + if missing: + # deduplicate + missing = set(Language.fromietf(la) for la in set(str(l) for l in missing)) + return added_at, item_id, item_title, item, missing + + +def items_get_all_missing_subs(items, sleep_after_request=False): + missing = [] + for added_at, kind, section_title, key in items: + try: + state = item_discover_missing_subs( + key, + kind=kind, + added_at=added_at, + section_title=section_title, + languages=config.lang_list.copy(), + internal=cast_bool(Prefs["subtitles.scan.embedded"]), + external=cast_bool(Prefs["subtitles.scan.external"]) + ) + if state: + # (added_at, item_id, title, item, missing_languages) + missing.append(state) + except: + Log.Error("Something went wrong when getting the state of item %s: %s", key, traceback.format_exc()) + if sleep_after_request: + time.sleep(sleep_after_request) + return missing + + +def refresh_item(item): + if not config.no_refresh: + Plex["library/metadata"].refresh(item) + + diff --git a/Contents/Code/support/plex_media.py b/Contents/Code/support/plex_media.py new file mode 100644 index 000000000..b4e4dacb0 --- /dev/null +++ b/Contents/Code/support/plex_media.py @@ -0,0 +1,438 @@ +# coding=utf-8 + +import os +import subprocess + +import helpers +from items import get_item +from subzero.language import Language +from lib import Plex +from support.config import TEXT_SUBTITLE_EXTS, config + + +def get_metadata_dict(item, part, add): + data = { + "item": item, + "section": item.section.title, + "path": part.file, + "folder": os.path.dirname(part.file), + "filename": os.path.basename(part.file) + } + data.update(add) + return data + + +imdb_guid_identifier = "com.plexapp.agents.imdb://" +tvdb_guid_identifier = "com.plexapp.agents.thetvdb://" + + +def get_plexapi_stream_info(plex_item, part_id=None): + if not plex_item: + return + + d = {"stream": {}} + data = d["stream"] + + # find current part + current_part = None + current_media = None + for media in plex_item.media: + for part in media.parts: + if not part_id or str(part.id) == part_id: + current_part = part + current_media = media + break + if current_part: + break + + if not current_part: + return d + + data["video_codec"] = current_media.video_codec + if current_media.audio_codec: + data["audio_codec"] = current_media.audio_codec.upper() + + if data["audio_codec"] == "DCA": + data["audio_codec"] = "DTS" + + if current_media.audio_channels == 8: + data["audio_channels"] = "7.1" + + elif current_media.audio_channels == 6: + data["audio_channels"] = "5.1" + else: + data["audio_channels"] = "%s.0" % str(current_media.audio_channels) + + # iter streams + for stream in current_part.streams: + if stream.stream_type == 1: + # video stream + data["resolution"] = "%s%s" % (current_media.video_resolution, + "i" if stream.scan_type != "progressive" else "p") + break + + return d + + +def media_to_videos(media, kind="series"): + """ + iterates through media and returns the associated parts (videos) + :param media: + :param kind: + :return: + """ + videos = [] + + # this is a Show or a Movie object + plex_item = get_item(media.id) + year = plex_item.year + original_title = plex_item.title_original + + if kind == "series": + for season in media.seasons: + season_object = media.seasons[season] + for episode in media.seasons[season].episodes: + ep = media.seasons[season].episodes[episode] + + tvdb_id = None + series_tvdb_id = None + if tvdb_guid_identifier in ep.guid: + tvdb_id = ep.guid[len(tvdb_guid_identifier):].split("?")[0] + series_tvdb_id = tvdb_id.split("/")[0] + + # get plex item via API for additional metadata + plex_episode = get_item(ep.id) + stream_info = get_plexapi_stream_info(plex_episode) + + if not stream_info: + continue + + for item in media.seasons[season].episodes[episode].items: + for part in item.parts: + videos.append( + get_metadata_dict(plex_episode, part, + dict(stream_info, **{"plex_part": part, "type": "episode", + "title": ep.title, + "series": media.title, "id": ep.id, "year": year, + "series_id": media.id, + "super_thumb": plex_item.thumb, + "season_id": season_object.id, + "imdb_id": None, "series_tvdb_id": series_tvdb_id, + "tvdb_id": tvdb_id, + "original_title": original_title, + "episode": plex_episode.index, + "season": plex_episode.season.index, + "section": plex_episode.section.title + }) + ) + ) + else: + stream_info = get_plexapi_stream_info(plex_item) + + if stream_info: + imdb_id = None + if imdb_guid_identifier in media.guid: + imdb_id = media.guid[len(imdb_guid_identifier):].split("?")[0] + for item in media.items: + for part in item.parts: + videos.append( + get_metadata_dict(plex_item, part, dict(stream_info, **{"plex_part": part, "type": "movie", + "title": media.title, "id": media.id, + "super_thumb": plex_item.thumb, + "series_id": None, "year": year, + "season_id": None, "imdb_id": imdb_id, + "original_title": original_title, + "series_tvdb_id": None, "tvdb_id": None, + "section": plex_item.section.title}) + ) + ) + return videos + + +IGNORE_FN = ("subzero.ignore", ".subzero.ignore", ".nosz") + + +def get_stream_fps(streams): + """ + accepts a list of plex streams or a list of the plex api streams + """ + for stream in streams: + # video + stream_type = getattr(stream, "type", getattr(stream, "stream_type", None)) + if stream_type == 1: + return getattr(stream, "frameRate", getattr(stream, "frame_rate", "25.000")) + return "25.000" + + +def get_media_item_ids(media, kind="series"): + # fixme: does this work correctly for full series force-refreshes and its intents? + ids = [media.id] + if kind == "series": + for season in media.seasons: + for episode in media.seasons[season].episodes: + ids.append(media.seasons[season].episodes[episode].id) + + return ids + + +def get_all_parts(plex_item): + parts = [] + for media in plex_item.media: + parts += media.parts + + return parts + + +def update_stream_info(part): + try: + return update_stream_info_(part) + except: + Log.Exception("Getting Mediainfo failed for: %s", part.file) + + +def update_stream_info_(part): + if config.mediainfo_bin and part.container == "mp4": + cmdline = '%s --Inform="Text;-%%ID%%_%%Title%%" %s' % (config.mediainfo_bin, helpers.quote(part.file)) + result = subprocess.check_output(cmdline, stderr=subprocess.PIPE, shell=True) + if result: + try: + stream_titles = {} + for pair in result[1:].split("-"): + sid, title = pair.split("_") + stream_titles[int(sid.strip())] = title.strip() + except: + pass + else: + filled = [] + for stream in part.streams: + if stream.index is None: + Log.Debug("Found stream with no index: %r", stream) + + index = stream.index+1 if stream.index is not None else 1 + if index in stream_titles: + stream.title = stream_titles[index] + filled.append(index-1) + if filled: + Log.Debug("Filled missing MP4 stream title info for streams: %s", filled) + + +def is_stream_forced(stream): + stream_title = getattr(stream, "title", "") or "" + forced = getattr(stream, "forced", False) + if not forced and stream_title and "forced" in stream_title.strip().lower(): + forced = True + + return forced + + +def get_embedded_subtitle_streams(part, requested_language=None, skip_duplicate_unknown=True, skip_unknown=False): + streams = [] + streams_unknown = [] + all_streams = [] + has_unknown = False + found_requested_language = False + update_stream_info(part) + for stream in part.streams: + # subtitle stream + if stream.stream_type == 3 and not stream.stream_key and stream.codec in TEXT_SUBTITLE_EXTS: + is_forced = is_stream_forced(stream) + language = helpers.get_language_from_stream(stream.language_code) + if language: + language = Language.rebuild(language, forced=is_forced) + + is_unknown = False + found_requested_language = requested_language and requested_language == language + stream_data = None + + if not language: + # only consider first unknown subtitle stream + if config.treat_und_as_first: + if has_unknown and skip_duplicate_unknown: + Log.Debug("skipping duplicate unknown") + continue + + language = Language.rebuild(list(config.lang_list)[0], forced=is_forced) + else: + language = None + is_unknown = True + has_unknown = True + stream_data = {"stream": stream, "is_unknown": is_unknown, "language": language, + "is_forced": is_forced} + streams_unknown.append(stream_data) + + if not requested_language or found_requested_language: + stream_data = {"stream": stream, "is_unknown": is_unknown, "language": language, + "is_forced": is_forced} + streams.append(stream_data) + + if found_requested_language: + break + + if stream_data: + all_streams.append(stream_data) + + if requested_language: + if streams_unknown and not found_requested_language and not skip_unknown: + streams = streams_unknown + else: + streams = all_streams + + return streams + + +def get_part(plex_item, part_id): + for media in plex_item.media: + for part in media.parts: + if str(part.id) == str(part_id): + return part + + +def get_plex_metadata(rating_key, part_id, item_type, plex_item=None): + """ + uses the Plex 3rd party API accessor to get metadata information + + :param rating_key: movie or episode + :param part_id: + :param item_type: + :return: + """ + + if not plex_item: + plex_item = get_item(rating_key) + + if not plex_item: + return + + # find current part + current_part = get_part(plex_item, part_id) + + if not current_part: + raise helpers.PartUnknownException("Part unknown") + + stream_info = get_plexapi_stream_info(plex_item, part_id) + + if not stream_info: + return + + # get normalized metadata + # fixme: duplicated logic of media_to_videos + if item_type == "episode": + show = list(Plex["library"].metadata(plex_item.show.rating_key))[0] + year = show.year + tvdb_id = None + series_tvdb_id = None + original_title = show.title_original + if tvdb_guid_identifier in plex_item.guid: + tvdb_id = plex_item.guid[len(tvdb_guid_identifier):].split("?")[0] + series_tvdb_id = tvdb_id.split("/")[0] + metadata = get_metadata_dict(plex_item, current_part, + dict(stream_info, + **{"plex_part": current_part, "type": "episode", "title": plex_item.title, + "series": plex_item.show.title, "id": plex_item.rating_key, + "series_id": plex_item.show.rating_key, + "season_id": plex_item.season.rating_key, + "imdb_id": None, + "year": year, + "tvdb_id": tvdb_id, + "super_thumb": plex_item.show.thumb, + "series_tvdb_id": series_tvdb_id, + "original_title": original_title, + "season": plex_item.season.index, + "episode": plex_item.index + }) + ) + else: + imdb_id = None + original_title = plex_item.title_original + if imdb_guid_identifier in plex_item.guid: + imdb_id = plex_item.guid[len(imdb_guid_identifier):].split("?")[0] + metadata = get_metadata_dict(plex_item, current_part, + dict(stream_info, **{"plex_part": current_part, "type": "movie", + "title": plex_item.title, "id": plex_item.rating_key, + "series_id": None, + "season_id": None, + "imdb_id": imdb_id, + "year": plex_item.year, + "tvdb_id": None, + "super_thumb": plex_item.thumb, + "series_tvdb_id": None, + "original_title": original_title, + "season": None, + "episode": None, + "section": plex_item.section.title}) + ) + return metadata + + +def get_blacklist_from_part_map(video_part_map, languages): + from support.storage import get_subtitle_storage + subtitle_storage = get_subtitle_storage() + blacklist = [] + for video, part in video_part_map.iteritems(): + stored_subs = subtitle_storage.load_or_new(video.plexapi_metadata["item"]) + for language in languages: + current_bl, subs = stored_subs.get_blacklist(part.id, language) + if not current_bl: + continue + + blacklist = blacklist + [(str(a), str(b)) for a, b in current_bl.keys()] + + subtitle_storage.destroy() + + return blacklist + + +class PMSMediaProxy(object): + """ + Proxy object for getting data from a mediatree items "internally" via the PMS + + note: this could be useful later on: Media.TV_Show(getattr(Metadata, "_access_point"), id=XXXXXX) + """ + + def __init__(self, media_id): + self.mediatree = Media.TreeForDatabaseID(media_id) + + def get_part(self, part_id=None): + """ + walk the mediatree until the given part was found; if no part was given, return the first one + :param part_id: + :return: + """ + m = self.mediatree + while 1: + if m.items: + media_item = m.items[0] + if not part_id: + return media_item.parts[0] if media_item.parts else None + + for part in media_item.parts: + if str(part.id) == str(part_id): + return part + break + + if not m.children: + break + + m = m.children[0] + + def get_all_parts(self): + """ + walk the mediatree until the given part was found; if no part was given, return the first one + :param part_id: + :return: + """ + m = self.mediatree + parts = [] + while 1: + if m.items: + media_item = m.items[0] + for part in media_item.parts: + parts.append(part) + break + + if not m.children: + break + + m = m.children[0] + return parts + diff --git a/Contents/Code/support/scanning.py b/Contents/Code/support/scanning.py new file mode 100644 index 000000000..f93dce055 --- /dev/null +++ b/Contents/Code/support/scanning.py @@ -0,0 +1,175 @@ +# coding=utf-8 +import traceback +import helpers +from babelfish.exceptions import LanguageError + +from support.lib import Plex, get_intent +from support.plex_media import get_stream_fps, is_stream_forced, update_stream_info +from support.storage import get_subtitle_storage +from support.config import config, TEXT_SUBTITLE_EXTS +from support.subtitlehelpers import get_subtitles_from_metadata +from subzero.video import parse_video, set_existing_languages +from subzero.language import language_from_stream, Language + + +def prepare_video(pms_video_info, ignore_all=False, hints=None, rating_key=None, providers=None, skip_hashing=False): + """ + returnes a subliminal/guessit-refined parsed video + :param pms_video_info: + :param ignore_all: + :param hints: + :param rating_key: + :return: + """ + embedded_subtitles = not ignore_all and Prefs['subtitles.scan.embedded'] + external_subtitles = not ignore_all and Prefs['subtitles.scan.external'] + + plex_part = pms_video_info["plex_part"] + + if ignore_all: + Log.Debug("Force refresh intended.") + + Log.Debug("Detecting streams: %s, account_for_external_subtitles=%s, account_for_embedded_subtitles=%s" % ( + plex_part.file, external_subtitles, embedded_subtitles)) + + known_embedded = [] + parts = [] + for media in list(Plex["library"].metadata(rating_key))[0].media: + parts += media.parts + + plexpy_part = None + for part in parts: + if int(part.id) == int(plex_part.id): + plexpy_part = part + + # embedded subtitles + # fixme: skip the whole scanning process if known_embedded == wanted languages? + audio_languages = [] + if plexpy_part: + update_stream_info(plexpy_part) + for stream in plexpy_part.streams: + if stream.stream_type == 2: + lang = None + try: + lang = language_from_stream(stream.language_code) + except LanguageError: + Log.Info("Couldn't detect embedded audio stream language: %s", stream.language_code) + + # treat unknown language as lang1? + if not lang and config.treat_und_as_first: + lang = Language.rebuild(list(config.lang_list)[0]) + Log.Info("Assuming language %s for audio stream: %s", lang, getattr(stream, "index", None)) + + audio_languages.append(lang) + + # subtitle stream + elif stream.stream_type == 3 and embedded_subtitles: + is_forced = is_stream_forced(stream) + + if ((config.forced_only or config.forced_also) and is_forced) or not is_forced: + # embedded subtitle + # fixme: tap into external subtitles here instead of scanning for ourselves later? + if stream.codec and getattr(stream, "index", None): + if config.exotic_ext or stream.codec.lower() in config.text_based_formats: + lang = None + try: + lang = language_from_stream(stream.language_code) + except LanguageError: + Log.Info("Couldn't detect embedded subtitle stream language: %s", stream.language_code) + + # treat unknown language as lang1? + if not lang and config.treat_und_as_first: + lang = Language.rebuild(list(config.lang_list)[0]) + Log.Info("Assuming language %s for subtitle stream: %s", lang, + getattr(stream, "index", None)) + + if lang: + if is_forced: + lang.forced = True + known_embedded.append(lang) + else: + Log.Warn("Part %s missing of %s, not able to scan internal streams", plex_part.id, rating_key) + + # metadata subtitles + known_metadata_subs = set() + meta_subs = get_subtitles_from_metadata(plex_part) + for language, subList in meta_subs.iteritems(): + try: + lang = Language.fromietf(Locale.Language.Match(language)) + except LanguageError: + if config.treat_und_as_first: + lang = Language.rebuild(list(config.lang_list)[0]) + else: + continue + + if subList: + for key in subList: + if key.startswith("subzero_md_forced"): + lang = Language.rebuild(lang, forced=True) + + known_metadata_subs.add(lang) + Log.Debug("Found metadata subtitle %r:%s for %s", lang, key, plex_part.file) + + Log.Debug("Known metadata subtitles: %r", known_metadata_subs) + Log.Debug("Known embedded subtitles: %r", known_embedded) + + subtitle_storage = get_subtitle_storage() + stored_subs = subtitle_storage.load(rating_key) + subtitle_storage.destroy() + + try: + # get basic video info scan (filename) + video = parse_video(plex_part.file, hints, skip_hashing=config.low_impact_mode or skip_hashing, + providers=providers) + + # set stream languages + if audio_languages: + video.audio_languages = audio_languages + Log.Info("Found audio streams: %s" % ", ".join([str(l) for l in audio_languages])) + + if not ignore_all: + set_existing_languages(video, pms_video_info, external_subtitles=external_subtitles, + embedded_subtitles=embedded_subtitles, known_embedded=known_embedded, + stored_subs=stored_subs, languages=config.lang_list, + only_one=config.only_one, known_metadata_subs=known_metadata_subs, + match_strictness=config.ext_match_strictness) + + # add video fps info + video.fps = plex_part.fps + return video + + except ValueError: + Log.Warn("File could not be guessed: %s: %s", plex_part.file, traceback.format_exc()) + + +def scan_videos(videos, ignore_all=False, providers=None, skip_hashing=False): + """ + receives a list of videos containing dictionaries returned by media_to_videos + :param videos: + :param kind: series or movies + :return: dictionary of subliminal.video.scan_video, key=subliminal scanned video, value=plex file part + """ + ret = {} + for video in videos: + intent = get_intent() + force_refresh = intent.get("force", video["id"], video["series_id"], video["season_id"]) + Log.Debug("Determining force-refresh (video: %s, series: %s, season: %s), result: %s" + % (video["id"], video["series_id"], video["season_id"], force_refresh)) + + hints = helpers.get_item_hints(video) + video["plex_part"].fps = get_stream_fps(video["plex_part"].streams) + p = providers or config.get_providers(media_type="series" if video["type"] == "episode" else "movies") + scanned_video = prepare_video(video, ignore_all=force_refresh or ignore_all, hints=hints, + rating_key=video["id"], providers=p, + skip_hashing=skip_hashing) + + if not scanned_video: + continue + + scanned_video.id = video["id"] + part_metadata = video.copy() + del part_metadata["plex_part"] + scanned_video.plexapi_metadata = part_metadata + scanned_video.ignore_all = force_refresh + ret[scanned_video] = video["plex_part"] + return ret diff --git a/Contents/Code/support/scheduler.py b/Contents/Code/support/scheduler.py new file mode 100644 index 000000000..f0f0d455d --- /dev/null +++ b/Contents/Code/support/scheduler.py @@ -0,0 +1,233 @@ +# coding=utf-8 + +import datetime +import logging +import traceback + +from config import config + +def parse_frequency(s): + if s == "never" or s is None: + return None, None + kind, num, unit = s.split() + return int(num), unit + + +class DefaultScheduler(object): + queue_thread = None + scheduler_thread = None + running = False + registry = None + + def __init__(self): + self.queue_thread = None + self.scheduler_thread = None + self.running = False + self.registry = [] + + self.tasks = {} + self.init_storage() + + def init_storage(self): + if "tasks" not in Dict: + Dict["tasks"] = {"queue": []} + Dict.Save() + + if "queue" not in Dict["tasks"]: + Dict["tasks"]["queue"] = [] + + def get_task_data(self, name): + if name not in Dict["tasks"]: + raise NotImplementedError("Task missing! %s" % name) + + if "data" in Dict["tasks"][name]: + return Dict["tasks"][name]["data"] + + def clear_task_data(self, name=None): + if name is None: + # full clean + Log.Debug("Clearing previous task data") + if Dict["tasks"]: + for task_name in Dict["tasks"].keys(): + if task_name == "queue": + Dict["tasks"][task_name] = [] + continue + + Dict["tasks"][task_name]["data"] = {} + Dict["tasks"][task_name]["running"] = False + Dict.Save() + return + + if name not in Dict["tasks"]: + raise NotImplementedError("Task missing! %s" % name) + + Dict["tasks"][name]["data"] = {} + Dict["tasks"][name]["running"] = False + Dict.Save() + Log.Debug("Task data cleared: %s", name) + + def register(self, task): + self.registry.append(task) + + def setup_tasks(self): + # discover tasks; + self.tasks = {} + for cls in self.registry: + task = cls() + try: + task_frequency = Prefs["scheduler.tasks.%s.frequency" % task.name] + except KeyError: + task_frequency = getattr(task, "frequency", None) + + self.tasks[task.name] = {"task": task, "frequency": parse_frequency(task_frequency)} + + def run(self): + self.running = True + self.scheduler_thread = Thread.Create(self.scheduler_worker) + self.queue_thread = Thread.Create(self.queue_worker) + + def stop(self): + self.running = False + + def task(self, name): + if name not in self.tasks: + return None + return self.tasks[name]["task"] + + def is_task_running(self, name): + task = self.task(name) + if task: + return task.running + + def last_run(self, task): + if task not in self.tasks: + return None + return self.tasks[task]["task"].last_run + + def next_run(self, task): + if task not in self.tasks or not self.tasks[task]["task"].periodic: + return None + frequency_num, frequency_key = self.tasks[task]["frequency"] + if not frequency_num: + return None + last = self.tasks[task]["task"].last_run + use_date = last + now = datetime.datetime.now() + if not use_date: + use_date = now + return max(use_date + datetime.timedelta(**{frequency_key: frequency_num}), now) + + def run_task(self, name, *args, **kwargs): + task = self.tasks[name]["task"] + + if task.running: + Log.Debug("Scheduler: Not running %s, as it's currently running.", name) + return False + + Log.Debug("Scheduler: Running task %s", name) + try: + task.prepare(*args, **kwargs) + task.run() + except Exception, e: + Log.Error("Scheduler: Something went wrong when running %s: %s", name, traceback.format_exc()) + finally: + try: + task.post_run(Dict["tasks"][name]["data"]) + except: + Log.Error("Scheduler: task.post_run failed for %s: %s", name, traceback.format_exc()) + Dict.Save() + config.sync_cache() + + def dispatch_task(self, *args, **kwargs): + if "queue" not in Dict["tasks"]: + Dict["tasks"]["queue"] = [] + + Dict["tasks"]["queue"].append((args, kwargs)) + + def signal(self, name, *args, **kwargs): + for task_name in self.tasks.keys(): + task = self.task(task_name) + if not task: + Log.Error("Scheduler: Task %s not found (?!)" % task_name) + continue + + if not task.periodic: + continue + + if task.running: + Log.Debug("Scheduler: Sending signal %s to task %s (%s, %s)", name, task_name, args, kwargs) + try: + status = task.signal(name, *args, **kwargs) + except NotImplementedError: + Log.Debug("Scheduler: Signal ignored by %s", task_name) + continue + if status: + Log.Debug("Scheduler: Signal accepted by %s", task_name) + else: + Log.Debug("Scheduler: Signal not accepted by %s", task_name) + continue + Log.Debug("Scheduler: Not sending signal %s to task %s, because: not running", name, task_name) + + def queue_worker(self): + Thread.Sleep(10.0) + while 1: + if not self.running: + break + + # single dispatch requested? + if Dict["tasks"]["queue"]: + # work queue off + queue = Dict["tasks"]["queue"][:] + Dict["tasks"]["queue"] = [] + Dict.Save() + for args, kwargs in queue: + Log.Debug("Queue: Dispatching single task: %s, %s", args, kwargs) + Thread.Create(self.run_task, True, *args, **kwargs) + Thread.Sleep(5.0) + + Thread.Sleep(1) + + def scheduler_worker(self): + Thread.Sleep(10.0) + while 1: + if not self.running: + break + + # scheduled tasks + for name in self.tasks.keys(): + now = datetime.datetime.now() + info = self.tasks.get(name) + if not info: + Log.Error("Scheduler: Task %s not found (?!)" % name) + continue + task = info["task"] + + if name not in Dict["tasks"] or not task.periodic: + continue + + if task.running: + continue + + frequency_num, frequency_key = info["frequency"] + if not frequency_num: + continue + + # run legacy SARAM once + if name == "SearchAllRecentlyAddedMissing" and ("hasRunLSARAM" not in Dict or not Dict["hasRunLSARAM"]): + task = self.tasks["LegacySearchAllRecentlyAddedMissing"]["task"] + task.last_run = None + name = "LegacySearchAllRecentlyAddedMissing" + Dict["hasRunLSARAM"] = True + Dict.Save() + + if not task.last_run or (task.last_run + datetime.timedelta(**{frequency_key: frequency_num}) <= now): + # fixme: scheduled tasks run synchronously. is this the best idea? + Thread.Create(self.run_task, True, name) + #Thread.Sleep(5.0) + #self.run_task(name) + Thread.Sleep(5.0) + + Thread.Sleep(1) + + +scheduler = DefaultScheduler() diff --git a/Contents/Code/support/storage.py b/Contents/Code/support/storage.py new file mode 100644 index 000000000..1adfa733c --- /dev/null +++ b/Contents/Code/support/storage.py @@ -0,0 +1,255 @@ +# coding=utf-8 + +import datetime +import os +import pprint +import copy +import traceback +import types + +from subliminal_patch.core import save_subtitles as subliminal_save_subtitles +from subzero.subtitle_storage import StoredSubtitlesManager +from subzero.lib.io import FileIO + +from subtitlehelpers import force_utf8 +from config import config +from helpers import notify_executable, get_title_for_video_metadata, cast_bool, force_unicode +from plex_media import PMSMediaProxy +from support.items import get_item + + +def get_subtitle_storage(): + return StoredSubtitlesManager(Data, Thread, get_item) + + +def store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage_type, mode="a", set_current=True): + """ + stores information about downloaded subtitles in plex's Dict() + """ + subtitle_storage = get_subtitle_storage() + for video, video_subtitles in downloaded_subtitles.items(): + part = scanned_video_part_map[video] + part_id = str(part.id) + video_id = str(video.id) + plex_item = get_item(video_id) + if not plex_item: + Log.Warn("Plex item not found: %s", video_id) + continue + + metadata = video.plexapi_metadata + title = get_title_for_video_metadata(metadata) + + stored_subs = subtitle_storage.load(video_id) + is_new = False + if not stored_subs: + is_new = True + Log.Debug(u"Creating new subtitle storage: %s, %s", video_id, part_id) + stored_subs = subtitle_storage.new(plex_item) + + for subtitle in video_subtitles: + lang = str(subtitle.language) + subtitle.normalize() + Log.Debug(u"Adding subtitle to storage: %s, %s, %s, %s, %s" % (video_id, part_id, lang, title, + subtitle.guess_encoding())) + + last_mod = None + if subtitle.storage_path: + last_mod = datetime.datetime.fromtimestamp(os.path.getmtime(subtitle.storage_path)) + + ret_val = stored_subs.add(part_id, lang, subtitle, storage_type, mode=mode, last_mod=last_mod, + set_current=set_current) + + if ret_val: + Log.Debug("Subtitle stored") + + else: + Log.Debug("Subtitle already existing in storage") + + if is_new or video_subtitles: + Log.Debug("Saving subtitle storage for %s" % video_id) + subtitle_storage.save(stored_subs) + + subtitle_storage.destroy() + + +def reset_storage(key): + """ + resets the Dict[key] storage, thanks to https://docs.google.com/document/d/1hhLjV1pI-TA5y91TiJq64BdgKwdLnFt4hWgeOqpz1NA/edit# + We can't use the nice Plex interface for this, as it calls get multiple times before set + #Plex[":/plugins/*/prefs"].set("com.plexapp.agents.subzero", "reset_storage", False) + """ + + Log.Debug("resetting storage") + Dict[key] = {} + Dict.Save() + + +def log_storage(key): + if not key: + Log.Debug(pprint.pformat(getattr(Dict, "_dict"))) + if key in Dict: + Log.Debug(pprint.pformat(Dict[key])) + + +def get_target_folder(file_path): + fld = None + fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() \ + if Prefs["subtitles.save.subFolder.Custom"] else None + + if fld_custom or Prefs["subtitles.save.subFolder"] != "current folder": + # specific subFolder requested, create it if it doesn't exist + fld_base = os.path.split(file_path)[0] + if fld_custom: + if fld_custom.startswith("/"): + # absolute folder + fld = fld_custom + else: + fld = os.path.join(fld_base, fld_custom) + else: + fld = os.path.join(fld_base, Prefs["subtitles.save.subFolder"]) + fld = force_unicode(fld) + if not os.path.exists(fld): + os.makedirs(fld) + return fld + + +def save_subtitles_to_file(subtitles, tags=None): + for video, video_subtitles in subtitles.items(): + if not video_subtitles: + continue + + if not isinstance(video, types.StringTypes): + file_path = video.name + else: + file_path = video + + fld = get_target_folder(file_path) + subliminal_save_subtitles(file_path, video_subtitles, directory=fld, single=cast_bool(Prefs['subtitles.only_one']), + chmod=config.chmod, path_decoder=force_unicode, + debug_mods=config.debug_mods, formats=config.subtitle_formats, tags=tags) + return True + + +def save_subtitles_to_metadata(videos, subtitles): + for video, video_subtitles in subtitles.items(): + mediaPart = videos[video] + for subtitle in video_subtitles: + content = subtitle.get_modified_content(debug=config.debug_mods) + + if not isinstance(mediaPart, Framework.api.agentkit.MediaPart): + # we're being handed a Plex.py model instance here, not an internal PMS MediaPart object. + # get the correct one + mp = PMSMediaProxy(video.id).get_part(mediaPart.id) + else: + mp = mediaPart + pm = Proxy.Media(content, ext="srt", forced="1" if subtitle.language.forced else None) + new_key = "subzero_md" + ("_forced" if subtitle.language.forced else "") + lang = Locale.Language.Match(subtitle.language.alpha2) + + for key, proxy in getattr(mp.subtitles[lang], "_proxies").iteritems(): + if not proxy or not len(proxy) >= 5: + Log.Debug("Can't parse metadata: %s" % repr(proxy)) + continue + if proxy[0] == "Media": + if not key.startswith("subzero_"): + if key == "subzero": + Log.Debug("Removing legacy metadata subtitle for %s", lang) + del mp.subtitles[lang][key] + Log.Debug("Existing metadata subtitle for %s: %s", lang, key) + + Log.Debug("Adding metadata sub for %s: %s", lang, subtitle) + mp.subtitles[lang][new_key] = pm + return True + + +def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a", bare_save=False, mods=None, + set_current=True): + """ + + :param set_current: save the subtitle as the current one + :param scanned_video_part_map: + :param downloaded_subtitles: + :param mode: + :param bare_save: don't trigger anything; don't store information + :param mods: enabled mods + :return: + """ + meta_fallback = False + save_successful = False + + # big fixme: scanned_video_part_map isn't needed to the current extent. rewrite. + + if mods: + for video, video_subtitles in downloaded_subtitles.items(): + if not video_subtitles: + continue + + for subtitle in video_subtitles: + Log.Info("Applying mods: %s to %s", mods, subtitle) + subtitle.mods = mods + subtitle.plex_media_fps = video.fps + + storage = "metadata" + save_to_fs = cast_bool(Prefs['subtitles.save.filesystem']) + if save_to_fs: + storage = "filesystem" + + if set_current: + if save_to_fs: + try: + Log.Debug("Using filesystem as subtitle storage") + save_subtitles_to_file(downloaded_subtitles) + except OSError: + if cast_bool(Prefs["subtitles.save.metadata_fallback"]): + meta_fallback = True + storage = "metadata" + else: + raise + else: + save_successful = True + + if not save_to_fs or meta_fallback: + if meta_fallback: + Log.Debug("Using metadata as subtitle storage, because filesystem storage failed") + else: + Log.Debug("Using metadata as subtitle storage") + save_successful = save_subtitles_to_metadata(scanned_video_part_map, downloaded_subtitles) + + if not bare_save and save_successful and config.notify_executable: + notify_executable(config.notify_executable, scanned_video_part_map, downloaded_subtitles, storage) + + if (not bare_save and save_successful) or not set_current: + store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage, mode=mode, set_current=set_current) + + return save_successful + + +def get_pack_id(subtitle): + return "%s_%s" % (subtitle.provider_name, subtitle.numeric_id) + + +def get_pack_data(subtitle): + subtitle_id = get_pack_id(subtitle) + + archive = os.path.join(config.pack_cache_dir, subtitle_id + ".archive") + if os.path.isfile(archive): + Log.Info("Loading archive from pack cache: %s", subtitle_id) + try: + data = FileIO.read(archive, 'rb') + + return data + except: + Log.Error("Couldn't load archive from pack cache: %s: %s", subtitle_id, traceback.format_exc()) + + +def store_pack_data(subtitle, data): + subtitle_id = get_pack_id(subtitle) + + archive = os.path.join(config.pack_cache_dir, subtitle_id + ".archive") + + Log.Info("Storing archive in pack cache: %s", subtitle_id) + try: + FileIO.write(archive, data, 'wb') + + except: + Log.Error("Couldn't store archive in pack cache: %s: %s", subtitle_id, traceback.format_exc()) diff --git a/Contents/Code/support/subtitlehelpers.py b/Contents/Code/support/subtitlehelpers.py new file mode 100644 index 000000000..d2b165fd7 --- /dev/null +++ b/Contents/Code/support/subtitlehelpers.py @@ -0,0 +1,207 @@ +# coding=utf-8 + +import re, os +import helpers + +from config import config, SUBTITLE_EXTS, TEXT_SUBTITLE_EXTS +from bs4 import UnicodeDammit +from subzero.language import match_ietf_language + + +class SubtitleHelper(object): + def __init__(self, filename): + self.filename = filename + + +def subtitle_helpers(filename): + filename = helpers.unicodize(filename) + helper_classes = [DefaultSubtitleHelper] + + if helpers.cast_bool(Prefs["subtitles.scan.exotic_ext"]): + helper_classes.insert(0, VobSubSubtitleHelper) + + for cls in helper_classes: + if cls.is_helper_for(filename): + return cls(filename) + return None + + +##################################################################################################################### + +class VobSubSubtitleHelper(SubtitleHelper): + @classmethod + def is_helper_for(cls, filename): + (file, file_extension) = os.path.splitext(filename) + + # We only support idx (and maybe sub) + if not file_extension.lower() in ['.idx', '.sub']: + return False + + # If we've been given a sub, we only support it if there exists a matching idx file + return os.path.exists(file + '.idx') + + def process_subtitles(self, part): + + lang_sub_map = {} + + # We don't directly process the sub file, only the idx. Therefore if we are passed on of these files, we simply + # ignore it. + (file, ext) = os.path.splitext(self.filename) + if ext == '.sub': + return lang_sub_map + + # If we have an idx file, we need to confirm there is an identically names sub file before we can proceed. + sub_filename = file + ".sub" + if not os.path.exists(sub_filename): + return lang_sub_map + + Log('Attempting to parse VobSub file: ' + self.filename) + idx = Core.storage.load(os.path.join(self.filename)) + if idx.count('VobSub index file') == 0: + Log('The idx file does not appear to be a VobSub, skipping...') + return lang_sub_map + + languages = {} + language_index = 0 + basename = os.path.basename(self.filename) + for language in re.findall('\nid: ([A-Za-z]{2})', idx): + + if not languages.has_key(language): + languages[language] = [] + + Log('Found .idx subtitle file: ' + self.filename + ' language: ' + language + ' stream index: ' + str(language_index)) + languages[language].append(Proxy.LocalFile(self.filename, index=str(language_index), format="vobsub")) + language_index += 1 + + if not lang_sub_map.has_key(language): + lang_sub_map[language] = [] + lang_sub_map[language].append(basename) + + for language, subs in languages.items(): + part.subtitles[language][basename] = subs + + return lang_sub_map + + +##################################################################################################################### + + +class DefaultSubtitleHelper(SubtitleHelper): + @classmethod + def is_helper_for(cls, filename): + (file, file_extension) = os.path.splitext(filename) + return file_extension.lower()[1:] in SUBTITLE_EXTS + + def process_subtitles(self, part): + + lang_sub_map = {} + + if not os.path.exists(self.filename): + return lang_sub_map + + basename = os.path.basename(self.filename) + (file, ext) = os.path.splitext(self.filename) + + # Remove the initial '.' from the extension + ext = ext[1:] + + forced = '' + default = '' + split_tag = file.rsplit('.', 1) + if len(split_tag) > 1 and split_tag[1].lower() in ['forced', 'normal', 'default', 'embedded', 'embedded-forced', + 'custom']: + file = split_tag[0] + sub_tag = split_tag[1].lower() + # don't do anything with 'normal', we don't need it + if 'forced' in sub_tag: + forced = '1' + elif 'default' == sub_tag: + default = '1' + + # Attempt to extract the language from the filename (e.g. Avatar (2009).eng) + # IETF support thanks to + # https://github.com/hpsbranco/LocalMedia.bundle/commit/4fad9aefedece78a1fa96401304351347f644369 + lang_part = match_ietf_language(file, ietf=helpers.cast_bool(Prefs["subtitles.language.ietf_display"])) + if lang_part != file: + language = Locale.Language.Match(lang_part) + elif config.only_one: + language = Locale.Language.Match(list(config.lang_list)[0].alpha2) + else: + language = Locale.Language.Match("xx") + + # skip non-SRT if wanted + if not config.exotic_ext and ext not in TEXT_SUBTITLE_EXTS: + return lang_sub_map + + codec = None + format = None + if ext in ['txt', 'sub']: + try: + + file_contents = Core.storage.load(self.filename) + lines = [line.strip() for line in file_contents.splitlines(True)] + if re.match('^\{[0-9]+\}\{[0-9]*\}', lines[1]): + format = 'microdvd' + elif re.match('^[0-9]{1,2}:[0-9]{2}:[0-9]{2}[:=,]', lines[1]): + format = 'txt' + elif '[SUBTITLE]' in lines[1]: + format = 'subviewer' + else: + Log("The subtitle file does not have a known format, skipping... : " + self.filename) + return lang_sub_map + except: + Log("An error occurred while attempting to parse the subtitle file, skipping... : " + self.filename) + return lang_sub_map + + # fixme: re-add vtt once Plex Inc. fixes this line in LocalMedia.bundle + if codec is None and ext in ['ass', 'ssa', 'smi', 'srt', 'psb']: + codec = ext.replace('ass', 'ssa') + + if format is None: + format = codec + + Log('Found subtitle file: ' + self.filename + ' language: ' + language + ' codec: ' + str( + codec) + ' format: ' + str(format) + ' default: ' + default + ' forced: ' + forced) + key = ("subzero_ex" + "_forced" if forced else "") + basename + part.subtitles[language][key] = Proxy.LocalFile(self.filename, codec=codec, format=format, default=default, + forced=forced) + + lang_sub_map[language] = [key] + return lang_sub_map + + +def get_subtitles_from_metadata(part): + subs = {} + if hasattr(part, "subtitles") and part.subtitles: + for language in part.subtitles: + subs[language] = [] + for key, proxy in getattr(part.subtitles[language], "_proxies").iteritems(): + if not proxy or not len(proxy) >= 5: + Log.Debug("Can't parse metadata: %s" % repr(proxy)) + continue + + p_type = proxy[0] + + if p_type == "Media": + if not key.startswith("subzero"): + continue + + # metadata subtitle + #Log.Debug(u"Found metadata subtitle: %s, %s, %s" % (language, key, repr(proxy))) + subs[language].append(key) + return subs + + +def force_utf8(content): + a = UnicodeDammit(content) + + if a.original_encoding: + Log.Debug("detected encoding: %s (None: most likely already successfully decoded)" % a.original_encoding) + else: + Log.Debug("detected encoding: unicode (already decoded)") + + # easy way out - already utf-8 + if a.original_encoding and a.original_encoding == "utf-8": + return content + + return (a.unicode_markup if a.unicode_markup else content.decode('ascii', 'replace')).encode("utf-8") diff --git a/Contents/Code/support/tasks.py b/Contents/Code/support/tasks.py new file mode 100755 index 000000000..24239b9f3 --- /dev/null +++ b/Contents/Code/support/tasks.py @@ -0,0 +1,944 @@ +# coding=utf-8 +import glob +import os +import datetime +import operator +import traceback +from urllib2 import URLError + +from subliminal_patch.score import compute_score +from subliminal_patch.core import download_subtitles +from subliminal import list_subtitles as list_all_subtitles, region as subliminal_cache_region +from subzero.language import Language +from subzero.video import refine_video + +from missing_subtitles import items_get_all_missing_subs, refresh_item +from scheduler import scheduler +from storage import save_subtitles, get_subtitle_storage +from support.config import config +from support.items import get_recent_items, get_item, is_wanted, get_item_title +from support.helpers import track_usage, get_title_for_video_metadata, cast_bool, PartUnknownException +from support.plex_media import get_plex_metadata +from support.extract import agent_extract_embedded +from support.scanning import scan_videos +from support.i18n import _ +from download import download_best_subtitles, pre_download_hook, post_download_hook, language_hook + + +class Task(object): + name = None + scheduler = None + periodic = False + running = False + time_start = None + data = None + + PROVIDER_SLACK = 30 + DL_PROVIDER_SLACK = 30 + + stored_attributes = ("last_run", "last_run_time", "running") + default_data = {"last_run": None, "last_run_time": None, "running": False, "data": {}} + + # task ready for being status-displayed? + ready_for_display = False + + def __init__(self): + self.name = self.get_class_name() + self.ready_for_display = False + self.time_start = None + self.setup_defaults() + + self.running = False + + def get_class_name(self): + return getattr(getattr(self, "__class__"), "__name__") + + def __getattribute__(self, name): + if name in object.__getattribute__(self, "stored_attributes"): + return Dict["tasks"].get(self.name, {}).get(name, None) + + return object.__getattribute__(self, name) + + def __setattr__(self, name, value): + if name in object.__getattribute__(self, "stored_attributes"): + Dict["tasks"][self.name][name] = value + Dict.Save() + return + + object.__setattr__(self, name, value) + + def setup_defaults(self): + if self.name not in Dict["tasks"]: + Dict["tasks"][self.name] = self.default_data.copy() + return + + sd = Dict["tasks"][self.name] + + # forward-migration + for key, def_value in self.default_data.iteritems(): + hasval = key in sd + if not hasval: + sd[key] = def_value + + def signal(self, *args, **kwargs): + raise NotImplementedError + + def prepare(self, *args, **kwargs): + return + + def run(self): + Log.Info(u"Task: running: %s", self.name) + self.time_start = datetime.datetime.now() + + def post_run(self, data_holder): + self.running = False + self.last_run = datetime.datetime.now() + if self.time_start and self.last_run: + self.last_run_time = self.last_run - self.time_start + self.time_start = None + Log.Info(u"Task: ran: %s", self.name) + + +class SubtitleListingMixin(object): + def list_subtitles(self, rating_key, item_type, part_id, language, skip_wrong_fps=True, metadata=None, + scanned_parts=None, air_date_cutoff=None): + + if not metadata: + metadata = get_plex_metadata(rating_key, part_id, item_type) + + if not metadata: + return + + providers = config.get_providers(media_type="series" if item_type == "episode" else "movies") + if not scanned_parts: + scanned_parts = scan_videos([metadata], ignore_all=True, providers=providers) + if not scanned_parts: + Log.Error(u"%s: Couldn't list available subtitles for %s", self.name, rating_key) + return + + video, plex_part = scanned_parts.items()[0] + refine_video(video, refiner_settings=config.refiner_settings) + + if air_date_cutoff is not None and metadata["item"].year and \ + metadata["item"].year + air_date_cutoff < datetime.date.today().year: + Log.Debug("Skipping searching for subtitles: %s, it aired over %s year(s) ago.", rating_key, + air_date_cutoff) + return + + config.init_subliminal_patches() + + provider_settings = config.provider_settings + #if not skip_wrong_fps: + # provider_settings["opensubtitlescom"]["skip_wrong_fps"] = False + + if item_type == "episode": + min_score = 240 + if video.is_special: + min_score = 180 + else: + min_score = 60 + + languages = {Language.fromietf(language)} + + available_subs = list_all_subtitles([video], languages, + providers=providers, + provider_configs=provider_settings, + pool_class=config.provider_pool, + throttle_callback=config.provider_throttle, + language_hook=language_hook) + + use_hearing_impaired = Prefs['subtitles.search.hearingImpaired'] in ("prefer", "force HI") + + # sort subtitles by score + unsorted_subtitles = [] + for s in available_subs[video]: + Log.Debug(u"%s: Starting score computation for %s", self.name, s) + try: + matches = s.get_matches(video) + except AttributeError: + Log.Error(u"%s: Match computation failed for %s: %s", self.name, s, traceback.format_exc()) + continue + + # skip wrong season/episodes + if item_type == "episode": + can_verify_series = True + if not s.hash_verifiable and "hash" in matches: + can_verify_series = False + + if can_verify_series and not {"series", "season", "episode"}.issubset(matches): + if "series" not in matches: + s.wrong_series = True + else: + s.wrong_season_ep = True + + orig_matches = matches.copy() + score, score_without_hash = compute_score(matches, s, video, hearing_impaired=use_hearing_impaired) + + unsorted_subtitles.append( + (s, score, score_without_hash, matches, orig_matches)) + scored_subtitles = sorted(unsorted_subtitles, key=operator.itemgetter(1, 2), reverse=True) + + subtitles = [] + for subtitle, score, score_without_hash, matches, orig_matches in scored_subtitles: + # check score + if score < min_score and not subtitle.wrong_series: + Log.Info(u'%s: Score %d is below min_score (%d)', self.name, score, min_score) + continue + subtitle.score = score + subtitle.matches = matches + subtitle.part_id = part_id + subtitle.item_type = item_type + subtitles.append(subtitle) + return subtitles + + +class DownloadSubtitleMixin(object): + def download_subtitle(self, subtitle, rating_key, mode="m"): + from interface.menu_helpers import set_refresh_menu_state + + item_type = subtitle.item_type + part_id = subtitle.part_id + metadata = get_plex_metadata(rating_key, part_id, item_type) + providers = config.get_providers(media_type="series" if item_type == "episode" else "movies") + scanned_parts = scan_videos([metadata], ignore_all=True, providers=providers) + video, plex_part = scanned_parts.items()[0] + + pre_download_hook(subtitle) + + # downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]} + download_subtitles([subtitle], providers=providers, + provider_configs=config.provider_settings, + pool_class=config.provider_pool, throttle_callback=config.provider_throttle) + + post_download_hook(subtitle) + + # may be redundant + subtitle.pack_data = None + + download_successful = False + + if subtitle.content: + try: + save_subtitles(scanned_parts, {video: [subtitle]}, mode=mode, mods=config.default_mods) + if mode == "m": + Log.Debug(u"%s: Manually downloaded subtitle for: %s", self.name, rating_key) + track_usage("Subtitle", "manual", "download", 1) + elif mode == "b": + Log.Debug(u"%s: Downloaded better subtitle for: %s", self.name, rating_key) + track_usage("Subtitle", "better", "download", 1) + download_successful = True + refresh_item(rating_key) + + except: + Log.Error(u"%s: Something went wrong when downloading specific subtitle: %s", + self.name, traceback.format_exc()) + finally: + set_refresh_menu_state(None) + + if download_successful: + # store item in history + from support.history import get_history + item_title = get_title_for_video_metadata(metadata, add_section_title=False) + history = get_history() + history.add(item_title, video.id, section_title=video.plexapi_metadata["section"], + thumb=video.plexapi_metadata["super_thumb"], + subtitle=subtitle, + mode=mode) + history.destroy() + + # clear missing subtitles menu data + if not scheduler.is_task_running("MissingSubtitles"): + scheduler.clear_task_data("MissingSubtitles") + else: + set_refresh_menu_state(_(u"%(class_name)s: Subtitle download failed (%(item_id)s)", + class_name=self.name, + item_id=rating_key)) + return download_successful + + +class AvailableSubsForItem(SubtitleListingMixin, Task): + item_type = None + part_id = None + language = None + rating_key = None + + def prepare(self, *args, **kwargs): + self.item_type = kwargs.get("item_type") + self.part_id = kwargs.get("part_id") + self.language = kwargs.get("language") + self.rating_key = kwargs.get("rating_key") + + def setup_defaults(self): + super(AvailableSubsForItem, self).setup_defaults() + + # reset any previous data + Dict["tasks"][self.name]["data"] = {} + + def run(self): + super(AvailableSubsForItem, self).run() + self.running = True + try: + track_usage("Subtitle", "manual", "list", 1) + except: + Log.Error("Something went wrong with track_usage: %s", traceback.format_exc()) + + Log.Debug("Listing available subtitles for: %s", self.rating_key) + subs = self.list_subtitles(self.rating_key, self.item_type, self.part_id, self.language, skip_wrong_fps=False) + if not subs: + self.data = "found_none" + return + + # we can't have nasty unpicklable stuff like ZipFile, BytesIO etc in self.data + self.data = [s.make_picklable() for s in subs] + + def post_run(self, task_data): + super(AvailableSubsForItem, self).post_run(task_data) + # clean old data + for key in task_data.keys(): + if key != self.rating_key: + del task_data[key] + task_data.update({self.rating_key: {self.language: self.data}}) + + +class DownloadSubtitleForItem(DownloadSubtitleMixin, Task): + subtitle = None + rating_key = None + + def prepare(self, *args, **kwargs): + self.subtitle = kwargs["subtitle"] + self.rating_key = kwargs["rating_key"] + + def run(self): + super(DownloadSubtitleForItem, self).run() + self.running = True + self.download_subtitle(self.subtitle, self.rating_key) + self.running = False + + +class MissingSubtitles(Task): + rating_key = None + item_type = None + part_id = None + language = None + + def run(self): + super(MissingSubtitles, self).run() + self.running = True + self.data = [] + recent_items = get_recent_items() + if recent_items: + self.data = items_get_all_missing_subs(recent_items) + + def post_run(self, task_data): + super(MissingSubtitles, self).post_run(task_data) + task_data["missing_subtitles"] = self.data + + +class SearchAllRecentlyAddedMissing(Task): + periodic = True + + items_done = None + items_searching = None + percentage = 0 + + def __init__(self): + super(SearchAllRecentlyAddedMissing, self).__init__() + self.items_done = None + self.items_searching = None + self.percentage = 0 + + def signal_updated_metadata(self, *args, **kwargs): + return True + + def prepare(self): + self.items_done = 0 + self.items_searching = 0 + self.percentage = 0 + self.ready_for_display = True + + def run(self): + super(SearchAllRecentlyAddedMissing, self).run() + + self.running = True + self.prepare() + + from support.history import get_history + history = get_history() + + now = datetime.datetime.now() + min_score_series = int(Prefs["subtitles.search.minimumTVScore2"].strip()) + min_score_movies = int(Prefs["subtitles.search.minimumMovieScore2"].strip()) + series_providers = config.get_providers(media_type="series") + movie_providers = config.get_providers(media_type="movies") + + is_recent_str = Prefs["scheduler.item_is_recent_age"] + num, ident = is_recent_str.split() + + max_search_days = 0 + if ident == "days": + max_search_days = int(num) + elif ident == "weeks": + max_search_days = int(num) * 7 + + subtitle_storage = get_subtitle_storage() + recent_files = subtitle_storage.get_recent_files(age_days=max_search_days) + + self.items_searching = len(recent_files) + + download_count = 0 + videos_with_downloads = 0 + + config.init_subliminal_patches() + + Log.Info(u"%s: Searching for subtitles for %s items", self.name, self.items_searching) + + def skip_item(): + self.items_searching = self.items_searching - 1 + self.percentage = int(self.items_done * 100 / self.items_searching) if self.items_searching > 0 else 100 + + # search for subtitles in viable items + try: + for fn in recent_files: + stored_subs = subtitle_storage.load(filename=fn) + if not stored_subs: + Log.Debug("Skipping item %s because storage is empty", fn) + skip_item() + continue + + video_id = stored_subs.video_id + + # added_date <= max_search_days? + if stored_subs.added_at + datetime.timedelta(days=max_search_days) <= now: + Log.Debug("Skipping item %s because it's too old", video_id) + skip_item() + continue + + if stored_subs.item_type == "episode": + min_score = min_score_series + providers = series_providers + else: + min_score = min_score_movies + providers = movie_providers + + parts = [] + plex_item = get_item(video_id) + + if not plex_item: + Log.Info(u"%s: Item %s unknown, skipping", self.name, video_id) + skip_item() + continue + + if not is_wanted(video_id, item=plex_item): + skip_item() + continue + + for media in plex_item.media: + parts += media.parts + + downloads_per_video = 0 + hit_providers = False + for part in parts: + part_id = part.id + + try: + metadata = get_plex_metadata(video_id, part_id, stored_subs.item_type) + except PartUnknownException: + Log.Info(u"%s: Part %s:%s unknown, skipping", self.name, video_id, part_id) + continue + + if not metadata: + Log.Info(u"%s: Part %s:%s unknown, skipping", self.name, video_id, part_id) + continue + + Log.Debug(u"%s: Looking for missing subtitles: %s", self.name, get_item_title(plex_item)) + scanned_parts = scan_videos([metadata], providers=providers) + + # auto extract embedded + if config.embedded_auto_extract: + if config.plex_transcoder: + ts = agent_extract_embedded(scanned_parts, set_as_existing=True) + if ts: + Log.Debug("Waiting for %i extraction threads to finish" % len(ts)) + for t in ts: + t.join() + else: + Log.Warn("Plex Transcoder not found, can't auto extract") + + downloaded_subtitles = download_best_subtitles(scanned_parts, min_score=min_score, + providers=providers) + hit_providers = downloaded_subtitles is not None + download_successful = False + + if downloaded_subtitles: + downloaded_any = any(downloaded_subtitles.values()) + if not downloaded_any: + continue + + try: + save_subtitles(scanned_parts, downloaded_subtitles, mode="a", mods=config.default_mods) + Log.Debug(u"%s: Downloaded subtitle for item with missing subs: %s", self.name, video_id) + download_successful = True + refresh_item(video_id) + track_usage("Subtitle", "manual", "download", 1) + except: + Log.Error(u"%s: Something went wrong when downloading specific subtitle: %s", self.name, + traceback.format_exc()) + finally: + scanned_parts = None + try: + item_title = get_title_for_video_metadata(metadata, add_section_title=False) + if download_successful: + # store item in history + for video, video_subtitles in downloaded_subtitles.items(): + if not video_subtitles: + continue + + for subtitle in video_subtitles: + downloads_per_video += 1 + history.add(item_title, video.id, section_title=metadata["section"], + thumb=video.plexapi_metadata["super_thumb"], + subtitle=subtitle, + mode="a") + + downloaded_subtitles = None + except: + Log.Error(u"%s: DEBUG HIT: %s", self.name, traceback.format_exc()) + + Log.Debug(u"%s: Waiting %s seconds before continuing", self.name, self.PROVIDER_SLACK) + Thread.Sleep(self.PROVIDER_SLACK) + + download_count += downloads_per_video + + if downloads_per_video: + videos_with_downloads += 1 + + self.items_done = self.items_done + 1 + self.percentage = int(self.items_done * 100 / self.items_searching) if self.items_searching > 0 else 100 + + stored_subs = None + + if downloads_per_video: + Log.Debug(u"%s: Subtitles have been downloaded, " + u"waiting %s seconds before continuing", self.name, self.DL_PROVIDER_SLACK) + Thread.Sleep(self.DL_PROVIDER_SLACK) + else: + if hit_providers: + Log.Debug(u"%s: Waiting %s seconds before continuing", self.name, self.PROVIDER_SLACK) + Thread.Sleep(self.PROVIDER_SLACK) + finally: + subtitle_storage.destroy() + history.destroy() + + if download_count: + Log.Debug(u"%s: done. Missing subtitles found for %s/%s items (%s subs downloaded)", self.name, + videos_with_downloads, self.items_searching, download_count) + else: + Log.Debug(u"%s: done. No subtitles found for %s items", self.name, self.items_searching) + + def post_run(self, task_data): + super(SearchAllRecentlyAddedMissing, self).post_run(task_data) + self.ready_for_display = False + self.percentage = 0 + self.items_done = None + self.items_searching = None + + +class LegacySearchAllRecentlyAddedMissing(Task): + periodic = True + frequency = "never" + items_done = None + items_searching = None + items_searching_ids = None + items_failed = None + percentage = 0 + + stall_time = 30 + + def __init__(self): + super(LegacySearchAllRecentlyAddedMissing, self).__init__() + self.items_done = None + self.items_searching = None + self.items_searching_ids = None + self.items_failed = None + self.percentage = 0 + + def signal(self, signal_name, *args, **kwargs): + handler = getattr(self, "signal_%s" % signal_name) + return handler(*args, **kwargs) if handler else None + + def signal_updated_metadata(self, *args, **kwargs): + item_id = int(args[0]) + + if self.items_searching_ids is not None and item_id in self.items_searching_ids: + self.items_done.append(item_id) + return True + + def prepare(self, *args, **kwargs): + self.items_done = [] + recent_items = get_recent_items() + missing = items_get_all_missing_subs(recent_items, sleep_after_request=0.2) + ids = set([id for added_at, id, title, item, missing_languages in missing if is_wanted(id, item=item)]) + self.items_searching = missing + self.items_searching_ids = ids + self.items_failed = [] + self.percentage = 0 + self.ready_for_display = True + + def run(self): + super(LegacySearchAllRecentlyAddedMissing, self).run() + self.running = True + missing_count = len(self.items_searching) + items_done_count = 0 + + for added_at, item_id, title, item, missing_languages in self.items_searching: + Log.Debug(u"Task: %s, triggering refresh for %s (%s)", self.name, title, item_id) + try: + refresh_item(item_id) + except URLError: + # timeout + pass + search_started = datetime.datetime.now() + tries = 1 + while 1: + if item_id in self.items_done: + items_done_count += 1 + self.percentage = int(items_done_count * 100 / missing_count) if missing_count > 0 else 100 + Log.Debug(u"Task: %s, item %s done (%s%%, %s/%s)", self.name, item_id, self.percentage, + items_done_count, missing_count) + break + + # item considered stalled after self.stall_time seconds passed after last refresh + if (datetime.datetime.now() - search_started).total_seconds() > self.stall_time: + if tries > 3: + self.items_failed.append(item_id) + Log.Debug(u"Task: %s, item stalled for %s times: %s, skipping", self.name, tries, item_id) + break + + Log.Debug(u"Task: %s, item stalled for %s seconds: %s, retrying", self.name, self.stall_time, + item_id) + tries += 1 + try: + refresh_item(item_id) + except URLError: + pass + search_started = datetime.datetime.now() + Thread.Sleep(1) + Thread.Sleep(0.1) + # we can't hammer the PMS, otherwise requests will be stalled + Thread.Sleep(5) + + Log.Debug("Task: %s, done (%s%%, %s/%s). Failed items: %s", self.name, self.percentage, + items_done_count, missing_count, self.items_failed) + + def post_run(self, task_data): + super(LegacySearchAllRecentlyAddedMissing, self).post_run(task_data) + self.ready_for_display = False + self.percentage = 0 + self.items_done = None + self.items_failed = None + self.items_searching = None + self.items_searching_ids = None + + +class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task): + periodic = True + + # TV: episode, format, series, year, season, video_codec, release_group, hearing_impaired, resolution + series_cutoff = 357 + + # movies: format, title, release_group, year, video_codec, resolution, hearing_impaired + movies_cutoff = 117 + + def signal_updated_metadata(self, *args, **kwargs): + return True + + def run(self): + super(FindBetterSubtitles, self).run() + self.running = True + better_found = 0 + try: + max_search_days = int(Prefs["scheduler.tasks.FindBetterSubtitles.max_days_after_added"].strip()) + except ValueError: + Log.Error(u"Please only put numbers into the FindBetterSubtitles.max_days_after_added setting. Exiting") + return + else: + if max_search_days > 30: + Log.Error(u"%s: FindBetterSubtitles.max_days_after_added is too big. Max is 30 days.", self.name) + return + + now = datetime.datetime.now() + min_score_series = int(Prefs["subtitles.search.minimumTVScore2"].strip()) + min_score_movies = int(Prefs["subtitles.search.minimumMovieScore2"].strip()) + min_score_extracted_series = config.advanced.find_better_as_extracted_tv_score or 352 + min_score_extracted_movies = config.advanced.find_better_as_extracted_movie_score or 112 + overwrite_manually_modified = cast_bool( + Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_modified"]) + overwrite_manually_selected = cast_bool( + Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected"]) + + air_date_cutoff_pref = Prefs["scheduler.tasks.FindBetterSubtitles.air_date_cutoff"] + if air_date_cutoff_pref == "don't limit": + air_date_cutoff = None + else: + air_date_cutoff = int(air_date_cutoff_pref.split()[0]) + + subtitle_storage = get_subtitle_storage() + viable_item_count = 0 + + try: + for fn in subtitle_storage.get_recent_files(age_days=max_search_days): + stored_subs = subtitle_storage.load(filename=fn) + if not stored_subs: + continue + + video_id = stored_subs.video_id + + if stored_subs.item_type == "episode": + cutoff = self.series_cutoff + min_score = min_score_series + min_score_extracted = min_score_extracted_series + else: + cutoff = self.movies_cutoff + min_score = min_score_movies + min_score_extracted = min_score_extracted_movies + + # don't search for better subtitles until at least 30 minutes have passed + if stored_subs.added_at + datetime.timedelta(minutes=30) > now: + Log.Debug(u"%s: Item %s too new, skipping", self.name, video_id) + continue + + # added_date <= max_search_days? + if stored_subs.added_at + datetime.timedelta(days=max_search_days) <= now: + continue + + viable_item_count += 1 + ditch_parts = [] + + # look through all stored subtitle data + for part_id, languages in stored_subs.parts.iteritems(): + part_id = str(part_id) + + # all languages + for language, current_subs in languages.iteritems(): + current_key = current_subs.get("current") + current = current_subs.get(current_key) + + # currently got subtitle? + # fixme: check for existence + if not current: + continue + current_score = current.score + current_mode = current.mode + + # late cutoff met? skip + if current_score >= cutoff: + Log.Debug(u"%s: Skipping finding better subs, " + u"cutoff met (current: %s, cutoff: %s): %s (%s)", + self.name, current_score, cutoff, stored_subs.title, video_id) + continue + + # got manual subtitle but don't want to touch those? + if current_mode == "m" and not overwrite_manually_selected: + Log.Debug(u"%s: Skipping finding better subs, " + u"had manual: %s (%s)", self.name, stored_subs.title, video_id) + continue + + # subtitle modifications different from default + if not overwrite_manually_modified and current.mods \ + and set(current.mods).difference(set(config.default_mods)): + Log.Debug(u"%s: Skipping finding better subs, it has manual modifications: %s (%s)", + self.name, stored_subs.title, video_id) + continue + + try: + subs = self.list_subtitles(video_id, stored_subs.item_type, part_id, language, + air_date_cutoff=air_date_cutoff) + except PartUnknownException: + Log.Info(u"%s: Part %s unknown/gone; ditching subtitle info", self.name, part_id) + ditch_parts.append(part_id) + continue + + hit_providers = subs is not None + + if subs: + # subs are already sorted by score + better_downloaded = False + better_tried_download = 0 + better_visited = 0 + for sub in subs: + if sub.score > current_score and sub.score > min_score: + if current.provider_name == "embedded" and sub.score < min_score_extracted: + Log.Debug(u"%s: Not downloading subtitle for %s, we've got an active extracted " + u"embedded sub and the min score %s isn't met (%s).", + self.name, video_id, min_score_extracted, sub.score) + better_visited += 1 + break + + Log.Debug(u"%s: Better subtitle found for %s, downloading", self.name, video_id) + better_tried_download += 1 + ret = self.download_subtitle(sub, video_id, mode="b") + if ret: + better_found += 1 + better_downloaded = True + break + else: + Log.Debug(u"%s: Couldn't download/save subtitle. " + u"Continuing to the next one", self.name) + Log.Debug(u"%s: Waiting %s seconds before continuing", + self.name, self.DL_PROVIDER_SLACK) + Thread.Sleep(self.DL_PROVIDER_SLACK) + better_visited += 1 + + if better_tried_download and not better_downloaded: + Log.Debug(u"%s: Tried downloading better subtitle for %s, " + u"but every try failed.", self.name, video_id) + + elif better_downloaded: + Log.Debug(u"%s: Better subtitle downloaded for %s", self.name, video_id) + + if better_tried_download or better_downloaded: + Log.Debug(u"%s: Waiting %s seconds before continuing", self.name, self.DL_PROVIDER_SLACK) + Thread.Sleep(self.DL_PROVIDER_SLACK) + + elif better_visited: + Log.Debug(u"%s: Waiting %s seconds before continuing", self.name, self.PROVIDER_SLACK) + Thread.Sleep(self.PROVIDER_SLACK) + + subs = None + + elif hit_providers: + # hit the providers but didn't try downloading? wait. + Log.Debug(u"%s: Waiting %s seconds before continuing", self.name, self.PROVIDER_SLACK) + Thread.Sleep(self.PROVIDER_SLACK) + + if ditch_parts: + for part_id in ditch_parts: + try: + del stored_subs.parts[part_id] + except KeyError: + pass + subtitle_storage.save(stored_subs) + ditch_parts = None + + stored_subs = None + + Thread.Sleep(1) + finally: + subtitle_storage.destroy() + + if better_found: + Log.Debug(u"%s: done. Better subtitles found for %s/%s items", self.name, better_found, + viable_item_count) + else: + Log.Debug(u"%s: done. No better subtitles found for %s items", self.name, viable_item_count) + + +class SubtitleStorageMaintenance(Task): + periodic = True + frequency = "every 7 days" + + def run(self): + super(SubtitleStorageMaintenance, self).run() + self.running = True + Log.Info(u"%s: Running subtitle storage maintenance", self.name) + storage = get_subtitle_storage() + try: + deleted_items = storage.delete_missing(wanted_languages=set(str(l) for l in config.lang_list)) + except OSError: + deleted_items = storage.delete_missing(wanted_languages=set(str(l) for l in config.lang_list), + scandir_generic=True) + + if deleted_items: + Log.Info(u"%s: Subtitle information for %d non-existant videos have been cleaned up", + self.name, len(deleted_items)) + Log.Debug(u"%s: Videos: %s", self.name, deleted_items) + else: + Log.Info(u"%s: Nothing to do", self.name) + + storage.destroy() + + +class MenuHistoryMaintenance(Task): + periodic = True + frequency = "every 7 days" + + def run(self): + super(MenuHistoryMaintenance, self).run() + self.running = True + Log.Info(u"%s: Running menu history maintenance", self.name) + now = datetime.datetime.now() + if "menu_history" in Dict: + for key, timeout in Dict["menu_history"].copy().items(): + if now > timeout: + try: + del Dict["menu_history"][key] + except: + pass + + +class MigrateSubtitleStorage(Task): + periodic = False + frequency = None + + def run(self): + super(MigrateSubtitleStorage, self).run() + self.running = True + Log.Info(u"%s: Running subtitle storage migration", self.name) + storage = get_subtitle_storage() + + def migrate(scandir_generic=False): + for fn in storage.get_all_files(scandir_generic=scandir_generic): + if fn.endswith(".json.gz"): + continue + Log.Debug(u"%s: Migrating %s", self.name, fn) + storage.load(None, fn) + + try: + migrate() + except OSError: + migrate(scandir_generic=True) + + storage.destroy() + + +class CacheMaintenance(Task): + periodic = True + frequency = "every 1 days" + + main_cache_validity = 14 # days + pack_cache_validity = 4 # days + + def run(self): + super(CacheMaintenance, self).run() + self.running = True + Log.Info(u"%s: Running cache maintenance", self.name) + now = datetime.datetime.now() + + def remove_expired(path, expiry): + mtime = datetime.datetime.fromtimestamp(os.path.getmtime(path)) + if mtime + datetime.timedelta(days=expiry) < now: + try: + os.remove(path) + except (IOError, OSError): + Log.Debug("Couldn't remove cache file: %s", os.path.basename(path)) + + # main cache + if config.new_style_cache: + for fn in subliminal_cache_region.backend.all_filenames: + remove_expired(fn, self.main_cache_validity) + + # archive cache + for fn in glob.iglob(os.path.join(config.pack_cache_dir, "*.archive")): + remove_expired(fn, self.pack_cache_validity) + + +scheduler.register(LegacySearchAllRecentlyAddedMissing) +scheduler.register(SearchAllRecentlyAddedMissing) +scheduler.register(AvailableSubsForItem) +scheduler.register(DownloadSubtitleForItem) +scheduler.register(MissingSubtitles) +scheduler.register(FindBetterSubtitles) +scheduler.register(SubtitleStorageMaintenance) +scheduler.register(MigrateSubtitleStorage) +scheduler.register(MenuHistoryMaintenance) +scheduler.register(CacheMaintenance) diff --git a/Contents/DefaultPrefs.json b/Contents/DefaultPrefs.json index 424b81c7b..397b50445 100644 --- a/Contents/DefaultPrefs.json +++ b/Contents/DefaultPrefs.json @@ -1,104 +1,1038 @@ [ - { - "id": "provider.addic7ed.username", - "label": "Addic7ed Username", - "type": "text", - "default": "Username" - }, - { - "id": "provider.addic7ed.password", - "label": "Addic7ed Password", - "type": "text", - "option": "hidden", - "default": "", - "secure": "true" - }, - { - "id": "langPref1", - "label": "Subtitle Language (1)", - "type": "enum", - "values": ["sq","ar","be","bs","bg","ca","zh","cs","da","nl","en","et","fi","fr","de","el","he","hi","hu","is","id","it","ja","ko","lv","lt","mk","ms","no","pl","pt","ro","ru","sr","sk","sl","es","sv","th","tr","uk","vi","hr"], - "default": "en" - }, - { - "id": "langPref2", - "label": "Subtitle Language (2)", - "type": "enum", - "values": ["None", "sq","ar","be","bs","bg","ca","zh","cs","da","nl","en","et","fi","fr","de","el","he","hi","hu","is","id","it","ja","ko","lv","lt","mk","ms","no","pl","pt","ro","ru","sr","sk","sl","es","sv","th","tr","uk","vi","hr"], - "default": "None" - }, - { - "id": "provider.opensubtitles.enabled", - "label": "Provider: Enable OpenSubtitles", - "type": "bool", - "default": "true" - }, - { - "id": "provider.thesubdb.enabled", - "label": "Provider: Enable TheSubDB", - "type": "bool", - "default": "true" - }, - { - "id": "provider.podnapisi.enabled", - "label": "Provider: Enable Podnapisi.NET", - "type": "bool", - "default": "true" - }, - { - "id": "provider.addic7ed.enabled", - "label": "Provider: Enable Addic7ed", - "type": "bool", - "default": "true" - }, - { - "id": "provider.tvsubtitles.enabled", - "label": "Provider: Enable TVsubtitles.net", - "type": "bool", - "default": "true" - }, - { - "id": "subtitles.scan.embedded", - "label": "Scan: include embedded subtitles", - "type": "bool", - "default": "false" - }, - { - "id": "subtitles.scan.external", - "label": "Scan: include external subtitles", - "type": "bool", - "default": "false" - }, - { - "id": "subtitles.search.minimumScore", - "label": "Minimum score for subtitles to download", - "type": "enum", - "values": ["100","95","90","85","80","75","70","65","60","55","50","45","40","35","30","25","20","15","10","5","0"], - "default": "0" - }, - { - "id": "subtitles.search.hearingImpaired", - "label": "Download hearing impaired subtitles.", - "type": "bool", - "default": "false" - }, - { - "id": "subtitles.save.filesystem", - "label": "Store subtitles next to media files (instead of metadata)", - "type": "bool", - "default": "false" - }, - { - "id": "subtitles.save.subFolder", - "label": "Subtitle Folder (\"current folder\" is the folder the current media file lives in)", - "type": "enum", - "values": ["current folder", "sub", "subs", "subtitle", "subtitles"], - "default": "current folder" - }, - { - "id": "subtitles.save.subFolder.Custom", - "label": "Custom Subtitle folder (computes to real paths; use for example \"bla\" as a subfolder of the current media file folder - can use real paths aswell)", - "type": "text", - "default": "" - }, + { + "id": "langPref1a", + "label": "Subtitle Language (1)", + "type": "enum", + "values": [ + "sq", + "ar", + "be", + "bs", + "bg", + "ca", + "zh", + "cs", + "da", + "nl", + "en", + "et", + "fi", + "fr", + "de", + "el", + "he", + "hi", + "hu", + "is", + "id", + "it", + "ja", + "ko", + "lv", + "lt", + "mk", + "ms", + "no", + "fa", + "pl", + "pt", + "pt-br", + "ro", + "ru", + "sr", + "sr-cyrl", + "sr-latn", + "sk", + "sl", + "es", + "sv", + "th", + "tr", + "uk", + "vi", + "hr", + "zh-hans", + "zh-hant" + ], + "default": "en" + }, + { + "id": "langPref2a", + "label": "Subtitle Language (2)", + "type": "enum", + "values": [ + "None", + "sq", + "ar", + "be", + "bs", + "bg", + "ca", + "zh", + "cs", + "da", + "nl", + "en", + "et", + "fi", + "fr", + "de", + "el", + "he", + "hi", + "hu", + "is", + "id", + "it", + "ja", + "ko", + "lv", + "lt", + "mk", + "ms", + "no", + "fa", + "pl", + "pt", + "pt-br", + "ro", + "ru", + "sr", + "sr-cyrl", + "sr-latn", + "sk", + "sl", + "es", + "sv", + "th", + "tr", + "uk", + "vi", + "hr", + "zh-hans", + "zh-hant" + ], + "default": "None" + }, + { + "id": "langPref3a", + "label": "Subtitle Language (3)", + "type": "enum", + "values": [ + "None", + "sq", + "ar", + "be", + "bs", + "bg", + "ca", + "zh", + "cs", + "da", + "nl", + "en", + "et", + "fi", + "fr", + "de", + "el", + "he", + "hi", + "hu", + "is", + "id", + "it", + "ja", + "ko", + "lv", + "lt", + "mk", + "ms", + "no", + "fa", + "pl", + "pt", + "pt-br", + "ro", + "ru", + "sr", + "sr-cyrl", + "sr-latn", + "sk", + "sl", + "es", + "sv", + "th", + "tr", + "uk", + "vi", + "hr", + "zh-hans", + "zh-hant" + ], + "default": "None" + }, + { + "id": "langPrefCustom", + "label": "Additional Subtitle Languages (use ISO-639-1 codes; comma-separated)", + "type": "text", + "default": "None" + }, + { + "id": "subtitles.when", + "label": "Download subtitles", + "type": "enum", + "values": [ + "Never", + "Always", + "When main audio stream is not Subtitle Language (1)", + "When main audio stream is not any configured language", + "When any audio stream is not Subtitle Language (1)", + "When any audio stream is not any configured language" + ], + "default": "Always" + }, + { + "id": "subtitles.ignore_for_audio", + "label": "Don't download subtitles for Audio languages (use ISO-639-1 codes; comma-separated; NULL=no audio)", + "type": "text", + "default": "None" + }, + { + "id": "subtitles.when_forced", + "label": "Download foreign/forced subtitles", + "type": "enum", + "values": [ + "Never", + "Always", + "Only for Subtitle Language (1)", + "Only for Subtitle Language (2)", + "Only for Subtitle Language (3)" + ], + "default": "Never" + }, + { + "id": "subtitles.any_language_is_enough", + "label": "Don't search for subtitles if a subtitle in any configured language exists as", + "type": "enum", + "values": [ + "External or embedded subtitle", + "External or embedded subtitle (not foreign/forced)", + "External subtitle", + "External subtitle (not foreign/forced)", + "Always search for all configured languages" + ], + "default": "Always search for all configured languages" + }, + { + "id": "subtitles.language.ietf_display", + "label": "Display languages with country attribute as ISO 639-1 (e.g. pt-BR = pt)", + "type": "bool", + "default": "true" + }, + { + "id": "subtitles.language.ietf_normalize", + "label": "Treat languages with country attribute as ISO 639-1 (e.g. don't download pt-BR if pt subtitle exists)", + "type": "bool", + "default": "false" + }, + { + "id": "subtitles.only_one", + "label": "Restrict to one language (skips adding \".lang.\" to the subtitle filename; only uses \"Subtitle Language (1)\")", + "type": "bool", + "default": "false" + }, + { + "id": "subtitles.language.treat_und_as_first", + "label": "Embedded streams: Treat \"Undefined\" (und) as language 1", + "type": "bool", + "default": "true" + }, + { + "id": "media_rename1", + "label": "I rename my files using", + "type": "enum", + "values": [ + "Sonarr/Radarr (fill api info below)", + "Filebot", + "Sonarr/Radarr/Filebot", + "Symlink to original file", + "I keep the original filenames", + "none of the above" + ], + "default": "I keep the original filenames" + }, + { + "id": "use_file_info_file", + "label": "Retrieve original filename from .file_info/file_info index files (see wiki)", + "type": "bool", + "default": "false" + }, + { + "id": "drone_api.sonarr.url", + "label": "Sonarr URL (add URL base if configured)", + "type": "text", + "default": "http://127.0.0.1:8989" + }, + { + "id": "drone_api.sonarr.api_key", + "label": "Sonarr API key", + "type": "text", + "default": "" + }, + { + "id": "drone_api.radarr.url", + "label": "Radarr URL (add URL base if configured, min. version: 0.2.0.897)", + "type": "text", + "default": "http://127.0.0.1:7878" + }, + { + "id": "drone_api.radarr.api_key", + "label": "Radarr API key", + "type": "text", + "default": "" + }, + { + "id": "anticaptcha.service", + "label": "AntiCaptcha-Service (needs paid account; enables Addic7ed)", + "type": "enum", + "values": [ + "none", + "anti-captcha.com", + "deathbycaptcha.com" + ], + "default": "none" + }, + { + "id": "anticaptcha.api_key", + "label": "AntiCaptcha-Service key (anti-captcha.com: account_key; deathbycaptcha.com: username:password)", + "type": "text", + "default": "" + }, + { + "id": "provider.opensubtitles.enabled", + "label": "Provider: Enable OpenSubtitles.com", + "type": "bool", + "default": "true" + }, + { + "id": "provider.opensubtitles.username", + "label": "Opensubtitles Username", + "type": "text", + "default": "" + }, + { + "id": "provider.opensubtitles.password", + "label": "Opensubtitles Password", + "type": "text", + "option": "hidden", + "default": "", + "secure": "true" + }, + { + "id": "provider.opensubtitles.use_hash", + "label": "OpenSubtitles hash?", + "type": "bool", + "default": "true" + }, + { + "id": "provider.opensubtitles.api_key", + "label": "OpenSubtitles APIKey", + "type": "text", + "default": "" + }, + { + "id": "provider.podnapisi.enabled", + "label": "Provider: Enable Podnapisi.NET", + "type": "bool", + "default": "true" + }, + { + "id": "provider.napisy24.enabled", + "label": "Provider: Enable Napisy24 (pl)", + "type": "bool", + "default": "false" + }, + { + "id": "provider.napisy24.username", + "label": "Napisy24 Username", + "type": "text", + "default": "" + }, + { + "id": "provider.napisy24.password", + "label": "Napisy24 Password", + "type": "text", + "option": "hidden", + "default": "", + "secure": "true" + }, + { + "id": "provider.addic7ed.enabled", + "label": "Provider: Enable Addic7ed (needs AntiCaptcha)", + "type": "bool", + "default": "true" + }, + { + "id": "provider.addic7ed.username", + "label": "Addic7ed Username", + "type": "text", + "default": "" + }, + { + "id": "provider.addic7ed.password", + "label": "Addic7ed Password", + "type": "text", + "option": "hidden", + "default": "", + "secure": "true" + }, + { + "id": "provider.addic7ed.is_vip", + "label": "Addic7ed VIP? (80 vs 40 downloads per day)", + "type": "bool", + "default": "false" + }, + { + "id": "provider.addic7ed.boost_by2", + "label": "Addic7ed: boost score (if requirements met)", + "type": "enum", + "values": [ + "100", + "95", + "90", + "85", + "80", + "75", + "70", + "67", + "65", + "60", + "55", + "50", + "45", + "40", + "35", + "30", + "25", + "21", + "20", + "19", + "15", + "10", + "5", + "0" + ], + "default": "19" + }, + { + "id": "provider.titlovi.enabled", + "label": "Provider: Enable Titlovi.com (User and Password required)", + "type": "bool", + "default": "true" + }, + { + "id": "provider.titlovi.username", + "label": "Titlovi Username", + "type": "text", + "default": "" + }, + { + "id": "provider.titlovi.password", + "label": "Titlovi Password", + "type": "text", + "option": "hidden", + "default": "", + "secure": "true" + }, + { + "id": "provider.legendastv.enabled", + "label": "Provider: Enable Legendas TV (mostly pt-BR; UNRAR NEEDED)", + "type": "bool", + "default": "false" + }, + { + "id": "provider.legendastv.username", + "label": "Legendas TV Username", + "type": "text", + "default": "" + }, + { + "id": "provider.legendastv.password", + "label": "Legendas TV Password", + "type": "text", + "option": "hidden", + "default": "", + "secure": "true" + }, + { + "id": "provider.tvsubtitles.enabled", + "label": "Provider: Enable TVsubtitles.net", + "type": "bool", + "default": "true" + }, + { + "id": "provider.napiprojekt.enabled", + "label": "Provider: Enable NapiProjekt.pl (Polish)", + "type": "bool", + "default": "false" + }, + { + "id": "provider.subscene.enabled", + "label": "Provider: Enable SubScene (TV shows)", + "type": "bool", + "default": "true" + }, + { + "id": "provider.subscene.username", + "label": "SubScene Username", + "type": "text", + "default": "" + }, + { + "id": "provider.subscene.password", + "label": "SubScene Password", + "type": "text", + "option": "hidden", + "default": "", + "secure": "true" + }, + { + "id": "provider.supersubtitles.enabled", + "label": "Provider: Enable feliratok.info (Hungarian)", + "type": "bool", + "default": "false" + }, + { + "id": "provider.hosszupuska.enabled", + "label": "Provider: Enable hosszupuskasub.com (Hungarian)", + "type": "bool", + "default": "false" + }, + { + "id": "provider.argenteam.enabled", + "label": "Provider: Enable aRGENTeaM (Spanish)", + "type": "bool", + "default": "false" + }, + { + "id": "provider.assrt.enabled", + "label": "Provider: Enable assrt.net (Chinese)", + "type": "bool", + "default": "false" + }, + { + "id": "provider.assrt.token", + "label": "Assrt API Token", + "type": "text", + "default": "" + }, + { + "id": "provider.bsplayer.enabled", + "label": "Provider: Enable BSPlayer Subtitles", + "type": "bool", + "default": "true" + }, + { + "id": "provider.wizdom.enabled", + "label": "Provider: Enable WizdomSubs (Hebrew)", + "type": "bool", + "default": "true" + }, + { + "id": "provider.ktuvit.enabled", + "label": "Provider: Enable Ktuvit (Hebrew)", + "type": "bool", + "default": "true" + }, + { + "id": "provider.ktuvit.username", + "label": "Ktuvit Username", + "type": "text", + "default": "" + }, + { + "id": "provider.ktuvit.password", + "label": "Ktuvit Password", + "type": "text", + "option": "hidden", + "secure": "true", + "default": "" + }, + { + "id": "providers.multithreading", + "label": "Search enabled providers simultaneously (multithreading)", + "type": "bool", + "default": "true" + }, + { + "id": "subtitles.embedded.autoextract", + "label": "Automatically extract and use embedded subtitles upon media addition (with configured default mods)", + "type": "bool", + "default": "false" + }, + { + "id": "subtitles.search_after_autoextract", + "label": "After automatic extraction of embedded subtitles, also immediately search for available subtitles?", + "type": "bool", + "default": "false" + }, + { + "id": "subtitles.scan.embedded", + "label": "Don't search for subtitles of a language if there are embedded subtitles inside the media file (MKV/MP4)?", + "type": "bool", + "default": "false" + }, + { + "id": "subtitles.scan.external", + "label": "Don't search for subtitles of a language if they already exist on the filesystem (metadata/filesystem)?", + "type": "bool", + "default": "true" + }, + { + "id": "subtitles.scan.filename_strictness", + "label": "How strict should these subtitles existing on the filesystem be detected?", + "type": "enum", + "values": [ + "exact: media filename match", + "loose: filename contains media filename", + "any" + ], + "default": "loose: filename contains media filename" + }, + { + "id": "subtitles.scan.exotic_ext", + "label": "Include non-text subtitle formats (anything else than .srt/.ssa/.ass/.vtt; embedded or external) in the above?", + "type": "bool", + "default": "false" + }, + { + "id": "subtitles.search.minimumTVScore2", + "label": "Minimum score for TV (min: 240, def/sane: 337, min-ideal: 352; see http://v.ht/szscores)", + "type": "text", + "default": "337" + }, + { + "id": "subtitles.search.minimumMovieScore2", + "label": "Minimum score for movies (min: 60, def/sane: 69, min-ideal: 82; see http://v.ht/szscores)", + "type": "text", + "default": "60" + }, + { + "id": "subtitles.search.hearingImpaired", + "label": "Download hearing impaired subtitles.", + "type": "enum", + "values": [ + "prefer", + "don't prefer", + "force HI", + "force non-HI" + ], + "default": "don't prefer" + }, + { + "id": "subtitles.remove_hi", + "label": "Remove Hearing Impaired tags from downloaded subtitles", + "type": "bool", + "default": "false" + }, + { + "id": "subtitles.remove_tags", + "label": "Remove style tags from downloaded subtitles (bold, italic, underline, colors, ...)", + "type": "bool", + "default": "false" + }, + { + "id": "subtitles.fix_common", + "label": "Fix common issues in subtitles", + "type": "bool", + "default": "true" + }, + { + "id": "subtitles.fix_ocr", + "label": "Fix common OCR errors in downloaded subtitles", + "type": "bool", + "default": "true" + }, + { + "id": "subtitles.fix_only_uppercase", + "label": "Fix only uppercase downloaded subtitles", + "type": "bool", + "default": "true" + }, + { + "id": "subtitles.reverse_rtl", + "label": "Reverse punctuation in RTL languages (heb, ara, fas)", + "type": "bool", + "default": "false" + }, + { + "id": "subtitles.colors", + "label": "Change colors of subtitles to", + "type": "enum", + "values": [ + "don't change", + "white", + "light-grey", + "red", + "green", + "yellow", + "blue", + "magenta", + "cyan", + "black", + "dark-red", + "dark-green", + "dark-yellow", + "dark-blue", + "dark-magenta", + "dark-cyan", + "dark-grey" + ], + "default": "don't change" + }, + { + "id": "subtitles.save.filesystem", + "label": "Store subtitles next to media files (instead of metadata)", + "type": "bool", + "default": "true" + }, + { + "id": "subtitles.save.formats", + "label": "Subtitle formats to save (non-SRT only works if the previous option is enabled)", + "type": "enum", + "values": [ + "SRT", + "VTT", + "SRT+VTT" + ], + "default": "SRT" + }, + { + "id": "subtitles.save.subFolder", + "label": "Subtitle Folder (\"current folder\" is the folder the current media file lives in)", + "type": "enum", + "values": [ + "current folder", + "sub", + "subs", + "subtitle", + "subtitles" + ], + "default": "current folder" + }, + { + "id": "subtitles.save.subFolder.Custom", + "label": "Custom Subtitle folder (overrides \"Subtitle Folder\"; computes to real paths)", + "type": "text", + "default": "" + }, + { + "id": "subtitles.save.metadata_fallback", + "label": "Fall back to metadata storage if filesystem storage failed", + "type": "bool", + "default": "false" + }, + { + "id": "subtitles.save.chmod", + "label": "Set subtitle file permissions to (integer, e.g.: 0775)", + "type": "text", + "default": "" + }, + { + "id": "subtitles.autoclean", + "label": "Automatically delete leftover/unused (externally saved) subtitles", + "type": "bool", + "default": "false" + }, + { + "id": "activity.on_playback", + "label": "On media playback: search for missing subtitles (refresh item)", + "type": "enum", + "values": [ + "never", + "current media item", + "next episode (series)", + "hybrid: current item or next episode", + "hybrid-plus: current item and next episode" + ], + "default": "never" + }, + { + "id": "scheduler.tasks.SearchAllRecentlyAddedMissing.frequency", + "label": "Scheduler: Periodically search for recent items with missing subtitles", + "type": "enum", + "values": [ + "never", + "every 6 hours", + "every 12 hours", + "every 24 hours" + ], + "default": "every 6 hours" + }, + { + "id": "scheduler.item_is_recent_age", + "label": "Scheduler: Item age to be considered recent", + "type": "enum", + "values": [ + "1 days", + "2 days", + "3 days", + "4 days", + "1 weeks", + "2 weeks", + "3 weeks", + "4 weeks", + "5 weeks", + "6 weeks", + "12 weeks" + ], + "default": "2 weeks" + }, + { + "id": "scheduler.max_recent_items_per_library", + "label": "Scheduler: Recent items to consider per library", + "type": "text", + "default": "1000" + }, + { + "id": "scheduler.tasks.FindBetterSubtitles.frequency", + "label": "Scheduler: Periodically search for better subtitles", + "type": "enum", + "values": [ + "never", + "every 6 hours", + "every 12 hours", + "every 24 hours" + ], + "default": "every 12 hours" + }, + { + "id": "scheduler.tasks.FindBetterSubtitles.max_days_after_added", + "label": "Scheduler: Days to search for better subtitles (max: 30 days)", + "type": "text", + "default": "7" + }, + { + "id": "scheduler.tasks.FindBetterSubtitles.air_date_cutoff", + "label": "Scheduler: Don't search for better subtitles if the item's air date is older than", + "type": "enum", + "values": [ + "don't limit", + "1 year", + "2 years", + "3 years", + "4 years", + "5 years", + "6 years", + "7 years", + "8 years", + "9 years", + "10 years" + ], + "default": "1 year" + }, + { + "id": "scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected", + "label": "Scheduler: Overwrite manually selected subtitles when better found", + "type": "bool", + "default": "true" + }, + { + "id": "scheduler.tasks.FindBetterSubtitles.overwrite_manually_modified", + "label": "Scheduler: Overwrite subtitles with non-default subtitle modifications when better found", + "type": "bool", + "default": "false" + }, + { + "id": "scheduler.tasks.SubtitleStorageMaintenance.frequency", + "label": "Scheduler: Periodically run subtitle storage maintenance (SZ internal)", + "type": "enum", + "values": [ + "never", + "every 6 hours", + "every 12 hours", + "every 24 hours", + "every 1 days", + "every 2 days", + "every 3 days", + "every 4 days", + "every 1 weeks", + "every 2 weeks", + "every 3 weeks", + "every 4 weeks", + "every 5 weeks", + "every 6 weeks", + "every 12 weeks" + ], + "default": "every 1 weeks" + }, + { + "id": "history_size", + "label": "History: amount of items to store historical data for", + "type": "enum", + "values": [ + "50", + "100", + "150", + "250", + "500" + ], + "default": "100" + }, + { + "id": "subtitles.try_downloads", + "label": "How many download tries per subtitle (on timeout or error)", + "type": "enum", + "values": [ + "1", + "2", + "3", + "4" + ], + "default": "2" + }, + { + "id": "subtitles.include_exclude_mode", + "label": "Should SZ be enabled or disabled by default? (impacts the settings below and the plugin menu)", + "type": "enum", + "values": [ + "enable SZ for all items by default, use ignore mode", + "disable SZ for all items by default, use include mode" + ], + "default": "enable SZ for all items by default, use ignore mode" + }, + { + "id": "subtitles.include_exclude_paths", + "label": "Enable/disable Sub-Zero in the following paths (comma-separated; the setting above impacts this)", + "type": "text", + "default": "" + }, + { + "id": "subtitles.include_exclude_fs", + "label": "Use \"subzero.ignore/.subzero.ignore/.nosz\" (ignore mode) or \"subzero.include/.subzero.include/.sz\" (include mode) files inside folders", + "type": "bool", + "default": "false" + }, + { + "id": "plugin_mode2", + "label": "Sub-Zero mode", + "type": "enum", + "values": [ + "agent + interface", + "only agent", + "only interface" + ], + "default": "agent + interface" + }, + { + "id": "plugin_pin", + "label": "Access PIN (any amount of numbers, 0-9)", + "type": "text", + "option": "hidden", + "default": "", + "secure": "true" + }, + { + "id": "plugin_pin_valid_for", + "label": "Access PIN valid for minutes", + "type": "text", + "default": "10" + }, + { + "id": "plugin_pin_mode2", + "label": "Use PIN to restrict access to (needs plugin or PMS restart)", + "type": "enum", + "values": [ + "disabled", + "interface", + "advanced menu" + ], + "default": "disabled" + }, + { + "id": "notify_executable", + "label": "Call this executable upon successful subtitle download (see Wiki for details)", + "type": "text", + "default": "" + }, + { + "id": "check_permissions", + "label": "Check for correct folder permissions of every library on plugin start", + "type": "bool", + "default": "true" + }, + { + "id": "new_style_cache", + "label": "Use new style caching (for subliminal)", + "type": "bool", + "default": "true" + }, + { + "id": "low_impact_mode", + "label": "Low impact mode (for remote filesystems)", + "type": "bool", + "default": "false" + }, + { + "id": "pms_request_timeout", + "label": "Timeout for API requests sent to the PMS", + "type": "text", + "default": "15" + }, + { + "id": "use_custom_dns2", + "label": "Use custom DNS (IPs, comma-separated, set to 'system' for system DNS. Default: Google/CF)", + "type": "text", + "default": "1.1.1.1, 8.8.8.8" + }, + { + "id": "proxy", + "label": "HTTP proxy to use for providers (supports credentials)", + "type": "text", + "default": "" + }, + { + "id": "path_to_advanced_settings", + "label": "Custom path to advanced_settings.json", + "type": "text", + "default": "" + }, + { + "id": "log_level", + "label": "How verbose should the logging be?", + "type": "enum", + "values": [ + "CRITICAL", + "ERROR", + "WARNING", + "INFO", + "DEBUG" + ], + "default": "WARNING" + }, + { + "id": "log_rotate_keep", + "label": "How many log backups to keep?", + "type": "text", + "default": "5" + }, + { + "id": "log_debug_mods", + "label": "Log subtitle modification (debug)", + "type": "bool", + "default": "false" + }, + { + "id": "log_console", + "label": "Log to console (for development/debugging)", + "type": "bool", + "default": "false" + }, + { + "id": "track_usage", + "label": "Collect anonymous usage statistics", + "type": "bool", + "default": "true" + } ] diff --git a/Contents/Info.plist b/Contents/Info.plist old mode 100644 new mode 100755 index de3843f34..76d8ca797 --- a/Contents/Info.plist +++ b/Contents/Info.plist @@ -4,30 +4,50 @@ <dict> <key>CFBundleDevelopmentRegion</key> <string>English</string> - <key>CFBundleExecutable</key> - <string>Test Plug-in</string> <key>CFBundleIdentifier</key> - <string>com.plexapp.agents.subliminal</string> + <string>com.plexapp.agents.subzero</string> <key>CFBundleInfoDictionaryVersion</key> <string>6.0</string> <key>CFBundleShortVersionString</key> - <string>1.0</string> + <string>2.6.5</string> <key>CFBundleSignature</key> <string>????</string> <key>CFBundleVersion</key> - <string>1.0</string> + <string>2.6.5.3280</string> <key>PlexFrameworkVersion</key> <string>2</string> <key>PlexPluginClass</key> <string>Agent</string> <key>PlexPluginMode</key> - <string>AlwaysOn</string> + <string>Daemon</string> <key>PlexPluginConsoleLogging</key> - <string>1</string> + <string>0</string> <key>PlexPluginDevMode</key> - <string>1</string> + <string>0</string> <key>PlexPluginCodePolicy</key> - <!-- this allows channels to access some python methods which are otherwise blocked, as well as import external code libraries, and interact with the PMS HTTP API --> + <!-- this allows channels to access some python methods which are otherwise blocked, as well as import external code libraries, and interact with the PMS HTTP API --> <string>Elevated</string> + <key>PlexAgentAttributionText</key> + <string><div style="white-space: pre;"><img src="https://raw.githubusercontent.com/pannal/Sub-Zero.bundle/master/Contents/Resources/subzero.gif" /> + +<h1>Sub-Zero for Plex</h1><i>Subtitles done right</i> + +Version 2.6.5.3280 + +Originally based on @bramwalet's awesome <a href="https://github.com/bramwalet/Subliminal.bundle">Subliminal.bundle</a> + +If you like this, buy me a beer: <a href="https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=G9VKR2B8PMNKG" target="_blank" title="donate"><img src="https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif" alt="donate" title="donate" /></a> + +<strong>Need help?</strong> +Wiki: <a href="http://v.ht/szwiki">http://v.ht/szwiki</a> +Score info: <a href="http://v.ht/szscores">http://v.ht/szscores</a> +Plex thread: <a href="https://forums.plex.tv/discussion/186575">https://forums.plex.tv/discussion/186575</a> +Github: <a href="https://github.com/pannal/Sub-Zero.bundle">https://github.com/pannal/Sub-Zero</a> + +3rd party licenses: <a href="https://github.com/pannal/Sub-Zero.bundle/tree/master/Licenses">https://github.com/pannal/Sub-Zero.bundle/tree/master/Licenses</a> + +panni, 2019 +</div> + </string> </dict> </plist> diff --git a/Contents/Libraries/Linux/aarch64/UnRAR/unrar b/Contents/Libraries/Linux/aarch64/UnRAR/unrar new file mode 100755 index 000000000..c5e244bda Binary files /dev/null and b/Contents/Libraries/Linux/aarch64/UnRAR/unrar differ diff --git a/Contents/Libraries/Linux/armv5tel/UnRAR/unrar b/Contents/Libraries/Linux/armv5tel/UnRAR/unrar new file mode 100755 index 000000000..c5e244bda Binary files /dev/null and b/Contents/Libraries/Linux/armv5tel/UnRAR/unrar differ diff --git a/Contents/Libraries/Linux/i386/UnRAR/unrar b/Contents/Libraries/Linux/i386/UnRAR/unrar new file mode 100755 index 000000000..b40268992 Binary files /dev/null and b/Contents/Libraries/Linux/i386/UnRAR/unrar differ diff --git a/Contents/Libraries/MacOSX/i386/UnRAR/unrar b/Contents/Libraries/MacOSX/i386/UnRAR/unrar new file mode 100755 index 000000000..f32104fbc Binary files /dev/null and b/Contents/Libraries/MacOSX/i386/UnRAR/unrar differ diff --git a/Contents/Libraries/Shared/_markerlib/__init__.py b/Contents/Libraries/Shared/_markerlib/__init__.py new file mode 100644 index 000000000..e2b237b1f --- /dev/null +++ b/Contents/Libraries/Shared/_markerlib/__init__.py @@ -0,0 +1,16 @@ +try: + import ast + from _markerlib.markers import default_environment, compile, interpret +except ImportError: + if 'ast' in globals(): + raise + def default_environment(): + return {} + def compile(marker): + def marker_fn(environment=None, override=None): + # 'empty markers are True' heuristic won't install extra deps. + return not marker.strip() + marker_fn.__doc__ = marker + return marker_fn + def interpret(marker, environment=None, override=None): + return compile(marker)() diff --git a/Contents/Libraries/Shared/_markerlib/markers.py b/Contents/Libraries/Shared/_markerlib/markers.py new file mode 100644 index 000000000..fa837061e --- /dev/null +++ b/Contents/Libraries/Shared/_markerlib/markers.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +"""Interpret PEP 345 environment markers. + +EXPR [in|==|!=|not in] EXPR [or|and] ... + +where EXPR belongs to any of those: + + python_version = '%s.%s' % (sys.version_info[0], sys.version_info[1]) + python_full_version = sys.version.split()[0] + os.name = os.name + sys.platform = sys.platform + platform.version = platform.version() + platform.machine = platform.machine() + platform.python_implementation = platform.python_implementation() + a free string, like '2.6', or 'win32' +""" + +__all__ = ['default_environment', 'compile', 'interpret'] + +import ast +import os +import platform +import sys +import weakref + +_builtin_compile = compile + +try: + from platform import python_implementation +except ImportError: + if os.name == "java": + # Jython 2.5 has ast module, but not platform.python_implementation() function. + def python_implementation(): + return "Jython" + else: + raise + + +# restricted set of variables +_VARS = {'sys.platform': sys.platform, + 'python_version': '%s.%s' % sys.version_info[:2], + # FIXME parsing sys.platform is not reliable, but there is no other + # way to get e.g. 2.7.2+, and the PEP is defined with sys.version + 'python_full_version': sys.version.split(' ', 1)[0], + 'os.name': os.name, + 'platform.version': platform.version(), + 'platform.machine': platform.machine(), + 'platform.python_implementation': python_implementation(), + 'extra': None # wheel extension + } + +for var in list(_VARS.keys()): + if '.' in var: + _VARS[var.replace('.', '_')] = _VARS[var] + +def default_environment(): + """Return copy of default PEP 385 globals dictionary.""" + return dict(_VARS) + +class ASTWhitelist(ast.NodeTransformer): + def __init__(self, statement): + self.statement = statement # for error messages + + ALLOWED = (ast.Compare, ast.BoolOp, ast.Attribute, ast.Name, ast.Load, ast.Str) + # Bool operations + ALLOWED += (ast.And, ast.Or) + # Comparison operations + ALLOWED += (ast.Eq, ast.Gt, ast.GtE, ast.In, ast.Is, ast.IsNot, ast.Lt, ast.LtE, ast.NotEq, ast.NotIn) + + def visit(self, node): + """Ensure statement only contains allowed nodes.""" + if not isinstance(node, self.ALLOWED): + raise SyntaxError('Not allowed in environment markers.\n%s\n%s' % + (self.statement, + (' ' * node.col_offset) + '^')) + return ast.NodeTransformer.visit(self, node) + + def visit_Attribute(self, node): + """Flatten one level of attribute access.""" + new_node = ast.Name("%s.%s" % (node.value.id, node.attr), node.ctx) + return ast.copy_location(new_node, node) + +def parse_marker(marker): + tree = ast.parse(marker, mode='eval') + new_tree = ASTWhitelist(marker).generic_visit(tree) + return new_tree + +def compile_marker(parsed_marker): + return _builtin_compile(parsed_marker, '<environment marker>', 'eval', + dont_inherit=True) + +_cache = weakref.WeakValueDictionary() + +def compile(marker): + """Return compiled marker as a function accepting an environment dict.""" + try: + return _cache[marker] + except KeyError: + pass + if not marker.strip(): + def marker_fn(environment=None, override=None): + """""" + return True + else: + compiled_marker = compile_marker(parse_marker(marker)) + def marker_fn(environment=None, override=None): + """override updates environment""" + if override is None: + override = {} + if environment is None: + environment = default_environment() + environment.update(override) + return eval(compiled_marker, environment) + marker_fn.__doc__ = marker + _cache[marker] = marker_fn + return _cache[marker] + +def interpret(marker, environment=None): + return compile(marker)(environment) diff --git a/Contents/Libraries/Shared/_scandir.c b/Contents/Libraries/Shared/_scandir.c new file mode 100644 index 000000000..295dedde5 --- /dev/null +++ b/Contents/Libraries/Shared/_scandir.c @@ -0,0 +1,1825 @@ +/* C speedups for scandir module + +This is divided into four sections (each prefixed with a "SECTION:" +comment): + +1) Python 2/3 compatibility +2) Helper utilities from posixmodule.c, fileutils.h, etc +3) SECTION: Main DirEntry and scandir implementation, taken from + Python 3.5's posixmodule.c +4) Module and method definitions and initialization code + +*/ + +#include <Python.h> +#include <structseq.h> +#include <structmember.h> +#include "osdefs.h" + +#ifdef MS_WINDOWS +#include <windows.h> +#include "winreparse.h" +#else +#include <dirent.h> +#ifndef HAVE_DIRENT_H +#define HAVE_DIRENT_H 1 +#endif +#endif + +#define MODNAME "scandir" + + +/* SECTION: Python 2/3 compatibility */ + +#if PY_MAJOR_VERSION >= 3 +#define INIT_ERROR return NULL +#else +#define INIT_ERROR return +// Because on PyPy, Py_FileSystemDefaultEncoding is (was) defined to be NULL +// (see PyPy Bitbucket issue #2669) +#define FS_ENCODING (Py_FileSystemDefaultEncoding ? Py_FileSystemDefaultEncoding : "UTF-8") +#endif + +#if PY_MAJOR_VERSION < 3 || PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION <= 2 +#define _Py_IDENTIFIER(name) static char * PyId_##name = #name; +#define _PyObject_GetAttrId(obj, pyid_name) PyObject_GetAttrString((obj), *(pyid_name)) +#define PyExc_FileNotFoundError PyExc_OSError +#define PyUnicode_AsUnicodeAndSize(unicode, addr_length) \ + PyUnicode_AsUnicode(unicode); *(addr_length) = PyUnicode_GetSize(unicode) +#endif + + +/* SECTION: Helper utilities from posixmodule.c, fileutils.h, etc */ + +#if !defined(MS_WINDOWS) && defined(DT_UNKNOWN) +#define HAVE_DIRENT_D_TYPE 1 +#endif + +#ifdef HAVE_DIRENT_H +#include <dirent.h> +#define NAMLEN(dirent) strlen((dirent)->d_name) +#else +#if defined(__WATCOMC__) && !defined(__QNX__) +#include <direct.h> +#define NAMLEN(dirent) strlen((dirent)->d_name) +#else +#define dirent direct +#define NAMLEN(dirent) (dirent)->d_namlen +#endif +#ifdef HAVE_SYS_NDIR_H +#include <sys/ndir.h> +#endif +#ifdef HAVE_SYS_DIR_H +#include <sys/dir.h> +#endif +#ifdef HAVE_NDIR_H +#include <ndir.h> +#endif +#endif + +#ifndef Py_CLEANUP_SUPPORTED +#define Py_CLEANUP_SUPPORTED 0x20000 +#endif + +#ifndef S_IFLNK +/* Windows doesn't define S_IFLNK but posixmodule.c maps + * IO_REPARSE_TAG_SYMLINK to S_IFLNK */ +# define S_IFLNK 0120000 +#endif + +// _Py_stat_struct is already defined in fileutils.h on Python 3.5+ +#if PY_MAJOR_VERSION < 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 5) +#ifdef MS_WINDOWS +struct _Py_stat_struct { + unsigned long st_dev; + unsigned __int64 st_ino; + unsigned short st_mode; + int st_nlink; + int st_uid; + int st_gid; + unsigned long st_rdev; + __int64 st_size; + time_t st_atime; + int st_atime_nsec; + time_t st_mtime; + int st_mtime_nsec; + time_t st_ctime; + int st_ctime_nsec; + unsigned long st_file_attributes; +}; +#else +# define _Py_stat_struct stat +#endif +#endif + +/* choose the appropriate stat and fstat functions and return structs */ +#undef STAT +#undef FSTAT +#undef STRUCT_STAT +#ifdef MS_WINDOWS +# define STAT win32_stat +# define LSTAT win32_lstat +# define FSTAT _Py_fstat_noraise +# define STRUCT_STAT struct _Py_stat_struct +#else +# define STAT stat +# define LSTAT lstat +# define FSTAT fstat +# define STRUCT_STAT struct stat +#endif + +#ifdef MS_WINDOWS + +static __int64 secs_between_epochs = 11644473600; /* Seconds between 1.1.1601 and 1.1.1970 */ + +static void +FILE_TIME_to_time_t_nsec(FILETIME *in_ptr, time_t *time_out, int* nsec_out) +{ + /* XXX endianness. Shouldn't matter, as all Windows implementations are little-endian */ + /* Cannot simply cast and dereference in_ptr, + since it might not be aligned properly */ + __int64 in; + memcpy(&in, in_ptr, sizeof(in)); + *nsec_out = (int)(in % 10000000) * 100; /* FILETIME is in units of 100 nsec. */ + *time_out = Py_SAFE_DOWNCAST((in / 10000000) - secs_between_epochs, __int64, time_t); +} + +/* Below, we *know* that ugo+r is 0444 */ +#if _S_IREAD != 0400 +#error Unsupported C library +#endif +static int +attributes_to_mode(DWORD attr) +{ + int m = 0; + if (attr & FILE_ATTRIBUTE_DIRECTORY) + m |= _S_IFDIR | 0111; /* IFEXEC for user,group,other */ + else + m |= _S_IFREG; + if (attr & FILE_ATTRIBUTE_READONLY) + m |= 0444; + else + m |= 0666; + return m; +} + +void +_Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, + struct _Py_stat_struct *result) +{ + memset(result, 0, sizeof(*result)); + result->st_mode = attributes_to_mode(info->dwFileAttributes); + result->st_size = (((__int64)info->nFileSizeHigh)<<32) + info->nFileSizeLow; + result->st_dev = info->dwVolumeSerialNumber; + result->st_rdev = result->st_dev; + FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_ctime, &result->st_ctime_nsec); + FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); + FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); + result->st_nlink = info->nNumberOfLinks; + result->st_ino = (((unsigned __int64)info->nFileIndexHigh)<<32) + info->nFileIndexLow; + if (reparse_tag == IO_REPARSE_TAG_SYMLINK) { + /* first clear the S_IFMT bits */ + result->st_mode ^= (result->st_mode & S_IFMT); + /* now set the bits that make this a symlink */ + result->st_mode |= S_IFLNK; + } + result->st_file_attributes = info->dwFileAttributes; +} + +static BOOL +get_target_path(HANDLE hdl, wchar_t **target_path) +{ + int buf_size, result_length; + wchar_t *buf; + + /* We have a good handle to the target, use it to determine + the target path name (then we'll call lstat on it). */ + buf_size = GetFinalPathNameByHandleW(hdl, 0, 0, + VOLUME_NAME_DOS); + if(!buf_size) + return FALSE; + + buf = PyMem_New(wchar_t, buf_size+1); + if (!buf) { + SetLastError(ERROR_OUTOFMEMORY); + return FALSE; + } + + result_length = GetFinalPathNameByHandleW(hdl, + buf, buf_size, VOLUME_NAME_DOS); + + if(!result_length) { + PyMem_Free(buf); + return FALSE; + } + + if(!CloseHandle(hdl)) { + PyMem_Free(buf); + return FALSE; + } + + buf[result_length] = 0; + + *target_path = buf; + return TRUE; +} + +static int +win32_get_reparse_tag(HANDLE reparse_point_handle, ULONG *reparse_tag) +{ + char target_buffer[MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; + REPARSE_DATA_BUFFER *rdb = (REPARSE_DATA_BUFFER *)target_buffer; + DWORD n_bytes_returned; + + if (0 == DeviceIoControl( + reparse_point_handle, + FSCTL_GET_REPARSE_POINT, + NULL, 0, /* in buffer */ + target_buffer, sizeof(target_buffer), + &n_bytes_returned, + NULL)) /* we're not using OVERLAPPED_IO */ + return FALSE; + + if (reparse_tag) + *reparse_tag = rdb->ReparseTag; + + return TRUE; +} + +static void +find_data_to_file_info_w(WIN32_FIND_DATAW *pFileData, + BY_HANDLE_FILE_INFORMATION *info, + ULONG *reparse_tag) +{ + memset(info, 0, sizeof(*info)); + info->dwFileAttributes = pFileData->dwFileAttributes; + info->ftCreationTime = pFileData->ftCreationTime; + info->ftLastAccessTime = pFileData->ftLastAccessTime; + info->ftLastWriteTime = pFileData->ftLastWriteTime; + info->nFileSizeHigh = pFileData->nFileSizeHigh; + info->nFileSizeLow = pFileData->nFileSizeLow; +/* info->nNumberOfLinks = 1; */ + if (pFileData->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + *reparse_tag = pFileData->dwReserved0; + else + *reparse_tag = 0; +} + +static BOOL +attributes_from_dir_w(LPCWSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *reparse_tag) +{ + HANDLE hFindFile; + WIN32_FIND_DATAW FileData; + hFindFile = FindFirstFileW(pszFile, &FileData); + if (hFindFile == INVALID_HANDLE_VALUE) + return FALSE; + FindClose(hFindFile); + find_data_to_file_info_w(&FileData, info, reparse_tag); + return TRUE; +} + +static int +win32_xstat_impl_w(const wchar_t *path, struct _Py_stat_struct *result, + BOOL traverse) +{ + int code; + HANDLE hFile, hFile2; + BY_HANDLE_FILE_INFORMATION info; + ULONG reparse_tag = 0; + wchar_t *target_path; + const wchar_t *dot; + + hFile = CreateFileW( + path, + FILE_READ_ATTRIBUTES, /* desired access */ + 0, /* share mode */ + NULL, /* security attributes */ + OPEN_EXISTING, + /* FILE_FLAG_BACKUP_SEMANTICS is required to open a directory */ + /* FILE_FLAG_OPEN_REPARSE_POINT does not follow the symlink. + Because of this, calls like GetFinalPathNameByHandle will return + the symlink path again and not the actual final path. */ + FILE_ATTRIBUTE_NORMAL|FILE_FLAG_BACKUP_SEMANTICS| + FILE_FLAG_OPEN_REPARSE_POINT, + NULL); + + if (hFile == INVALID_HANDLE_VALUE) { + /* Either the target doesn't exist, or we don't have access to + get a handle to it. If the former, we need to return an error. + If the latter, we can use attributes_from_dir. */ + if (GetLastError() != ERROR_SHARING_VIOLATION) + return -1; + /* Could not get attributes on open file. Fall back to + reading the directory. */ + if (!attributes_from_dir_w(path, &info, &reparse_tag)) + /* Very strange. This should not fail now */ + return -1; + if (info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + if (traverse) { + /* Should traverse, but could not open reparse point handle */ + SetLastError(ERROR_SHARING_VIOLATION); + return -1; + } + } + } else { + if (!GetFileInformationByHandle(hFile, &info)) { + CloseHandle(hFile); + return -1; + } + if (info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + if (!win32_get_reparse_tag(hFile, &reparse_tag)) + return -1; + + /* Close the outer open file handle now that we're about to + reopen it with different flags. */ + if (!CloseHandle(hFile)) + return -1; + + if (traverse) { + /* In order to call GetFinalPathNameByHandle we need to open + the file without the reparse handling flag set. */ + hFile2 = CreateFileW( + path, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, + NULL, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL|FILE_FLAG_BACKUP_SEMANTICS, + NULL); + if (hFile2 == INVALID_HANDLE_VALUE) + return -1; + + if (!get_target_path(hFile2, &target_path)) + return -1; + + code = win32_xstat_impl_w(target_path, result, FALSE); + PyMem_Free(target_path); + return code; + } + } else + CloseHandle(hFile); + } + _Py_attribute_data_to_stat(&info, reparse_tag, result); + + /* Set S_IEXEC if it is an .exe, .bat, ... */ + dot = wcsrchr(path, '.'); + if (dot) { + if (_wcsicmp(dot, L".bat") == 0 || _wcsicmp(dot, L".cmd") == 0 || + _wcsicmp(dot, L".exe") == 0 || _wcsicmp(dot, L".com") == 0) + result->st_mode |= 0111; + } + return 0; +} + +static int +win32_xstat_w(const wchar_t *path, struct _Py_stat_struct *result, BOOL traverse) +{ + /* Protocol violation: we explicitly clear errno, instead of + setting it to a POSIX error. Callers should use GetLastError. */ + int code = win32_xstat_impl_w(path, result, traverse); + errno = 0; + return code; +} + +static int +win32_lstat_w(const wchar_t* path, struct _Py_stat_struct *result) +{ + return win32_xstat_w(path, result, FALSE); +} + +static int +win32_stat_w(const wchar_t* path, struct _Py_stat_struct *result) +{ + return win32_xstat_w(path, result, TRUE); +} + +#endif /* MS_WINDOWS */ + +static PyTypeObject StatResultType; + +static PyObject *billion = NULL; + +static newfunc structseq_new; + +static PyObject * +statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyStructSequence *result; + int i; + + result = (PyStructSequence*)structseq_new(type, args, kwds); + if (!result) + return NULL; + /* If we have been initialized from a tuple, + st_?time might be set to None. Initialize it + from the int slots. */ + for (i = 7; i <= 9; i++) { + if (result->ob_item[i+3] == Py_None) { + Py_DECREF(Py_None); + Py_INCREF(result->ob_item[i]); + result->ob_item[i+3] = result->ob_item[i]; + } + } + return (PyObject*)result; +} + +/* If true, st_?time is float. */ +static int _stat_float_times = 1; + +static void +fill_time(PyObject *v, int index, time_t sec, unsigned long nsec) +{ +#if SIZEOF_TIME_T > SIZEOF_LONG + PyObject *s = PyLong_FromLongLong((PY_LONG_LONG)sec); +#else +#if PY_MAJOR_VERSION >= 3 + PyObject *s = PyLong_FromLong((long)sec); +#else + PyObject *s = PyInt_FromLong((long)sec); +#endif +#endif + PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); + PyObject *s_in_ns = NULL; + PyObject *ns_total = NULL; + PyObject *float_s = NULL; + + if (!(s && ns_fractional)) + goto exit; + + s_in_ns = PyNumber_Multiply(s, billion); + if (!s_in_ns) + goto exit; + + ns_total = PyNumber_Add(s_in_ns, ns_fractional); + if (!ns_total) + goto exit; + + if (_stat_float_times) { + float_s = PyFloat_FromDouble(sec + 1e-9*nsec); + if (!float_s) + goto exit; + } + else { + float_s = s; + Py_INCREF(float_s); + } + + PyStructSequence_SET_ITEM(v, index, s); + PyStructSequence_SET_ITEM(v, index+3, float_s); + PyStructSequence_SET_ITEM(v, index+6, ns_total); + s = NULL; + float_s = NULL; + ns_total = NULL; +exit: + Py_XDECREF(s); + Py_XDECREF(ns_fractional); + Py_XDECREF(s_in_ns); + Py_XDECREF(ns_total); + Py_XDECREF(float_s); +} + +#ifdef MS_WINDOWS +#define HAVE_STAT_NSEC 1 +#define HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES 1 +#endif + +#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE +#define ST_BLKSIZE_IDX 16 +#else +#define ST_BLKSIZE_IDX 15 +#endif + +#ifdef HAVE_STRUCT_STAT_ST_BLOCKS +#define ST_BLOCKS_IDX (ST_BLKSIZE_IDX+1) +#else +#define ST_BLOCKS_IDX ST_BLKSIZE_IDX +#endif + +#ifdef HAVE_STRUCT_STAT_ST_RDEV +#define ST_RDEV_IDX (ST_BLOCKS_IDX+1) +#else +#define ST_RDEV_IDX ST_BLOCKS_IDX +#endif + +#ifdef HAVE_STRUCT_STAT_ST_FLAGS +#define ST_FLAGS_IDX (ST_RDEV_IDX+1) +#else +#define ST_FLAGS_IDX ST_RDEV_IDX +#endif + +#ifdef HAVE_STRUCT_STAT_ST_GEN +#define ST_GEN_IDX (ST_FLAGS_IDX+1) +#else +#define ST_GEN_IDX ST_FLAGS_IDX +#endif + +#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME +#define ST_BIRTHTIME_IDX (ST_GEN_IDX+1) +#else +#define ST_BIRTHTIME_IDX ST_GEN_IDX +#endif + +#ifdef HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES +#define ST_FILE_ATTRIBUTES_IDX (ST_BIRTHTIME_IDX+1) +#else +#define ST_FILE_ATTRIBUTES_IDX ST_BIRTHTIME_IDX +#endif + +#ifdef HAVE_LONG_LONG +# define _PyLong_FromDev PyLong_FromLongLong +#else +# define _PyLong_FromDev PyLong_FromLong +#endif + +#ifndef MS_WINDOWS +PyObject * +_PyLong_FromUid(uid_t uid) +{ + if (uid == (uid_t)-1) + return PyLong_FromLong(-1); + return PyLong_FromUnsignedLong(uid); +} + +PyObject * +_PyLong_FromGid(gid_t gid) +{ + if (gid == (gid_t)-1) + return PyLong_FromLong(-1); + return PyLong_FromUnsignedLong(gid); +} +#endif + +/* pack a system stat C structure into the Python stat tuple + (used by posix_stat() and posix_fstat()) */ +static PyObject* +_pystat_fromstructstat(STRUCT_STAT *st) +{ + unsigned long ansec, mnsec, cnsec; + PyObject *v = PyStructSequence_New(&StatResultType); + if (v == NULL) + return NULL; + + PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long)st->st_mode)); +#ifdef HAVE_LARGEFILE_SUPPORT + PyStructSequence_SET_ITEM(v, 1, + PyLong_FromUnsignedLongLong(st->st_ino)); +#else + PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLong((unsigned long)st->st_ino)); +#endif +#ifdef MS_WINDOWS + PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLong(st->st_dev)); +#else + PyStructSequence_SET_ITEM(v, 2, _PyLong_FromDev(st->st_dev)); +#endif + PyStructSequence_SET_ITEM(v, 3, PyLong_FromLong((long)st->st_nlink)); +#if defined(MS_WINDOWS) + PyStructSequence_SET_ITEM(v, 4, PyLong_FromLong(0)); + PyStructSequence_SET_ITEM(v, 5, PyLong_FromLong(0)); +#else + PyStructSequence_SET_ITEM(v, 4, _PyLong_FromUid(st->st_uid)); + PyStructSequence_SET_ITEM(v, 5, _PyLong_FromGid(st->st_gid)); +#endif +#ifdef HAVE_LARGEFILE_SUPPORT + PyStructSequence_SET_ITEM(v, 6, + PyLong_FromLongLong((PY_LONG_LONG)st->st_size)); +#else + PyStructSequence_SET_ITEM(v, 6, PyLong_FromLong(st->st_size)); +#endif + +#if defined(HAVE_STAT_TV_NSEC) + ansec = st->st_atim.tv_nsec; + mnsec = st->st_mtim.tv_nsec; + cnsec = st->st_ctim.tv_nsec; +#elif defined(HAVE_STAT_TV_NSEC2) + ansec = st->st_atimespec.tv_nsec; + mnsec = st->st_mtimespec.tv_nsec; + cnsec = st->st_ctimespec.tv_nsec; +#elif defined(HAVE_STAT_NSEC) + ansec = st->st_atime_nsec; + mnsec = st->st_mtime_nsec; + cnsec = st->st_ctime_nsec; +#else + ansec = mnsec = cnsec = 0; +#endif + fill_time(v, 7, st->st_atime, ansec); + fill_time(v, 8, st->st_mtime, mnsec); + fill_time(v, 9, st->st_ctime, cnsec); + +#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE + PyStructSequence_SET_ITEM(v, ST_BLKSIZE_IDX, + PyLong_FromLong((long)st->st_blksize)); +#endif +#ifdef HAVE_STRUCT_STAT_ST_BLOCKS + PyStructSequence_SET_ITEM(v, ST_BLOCKS_IDX, + PyLong_FromLong((long)st->st_blocks)); +#endif +#ifdef HAVE_STRUCT_STAT_ST_RDEV + PyStructSequence_SET_ITEM(v, ST_RDEV_IDX, + PyLong_FromLong((long)st->st_rdev)); +#endif +#ifdef HAVE_STRUCT_STAT_ST_GEN + PyStructSequence_SET_ITEM(v, ST_GEN_IDX, + PyLong_FromLong((long)st->st_gen)); +#endif +#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME + { + PyObject *val; + unsigned long bsec,bnsec; + bsec = (long)st->st_birthtime; +#ifdef HAVE_STAT_TV_NSEC2 + bnsec = st->st_birthtimespec.tv_nsec; +#else + bnsec = 0; +#endif + if (_stat_float_times) { + val = PyFloat_FromDouble(bsec + 1e-9*bnsec); + } else { + val = PyLong_FromLong((long)bsec); + } + PyStructSequence_SET_ITEM(v, ST_BIRTHTIME_IDX, + val); + } +#endif +#ifdef HAVE_STRUCT_STAT_ST_FLAGS + PyStructSequence_SET_ITEM(v, ST_FLAGS_IDX, + PyLong_FromLong((long)st->st_flags)); +#endif +#ifdef HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES + PyStructSequence_SET_ITEM(v, ST_FILE_ATTRIBUTES_IDX, + PyLong_FromUnsignedLong(st->st_file_attributes)); +#endif + + if (PyErr_Occurred()) { + Py_DECREF(v); + return NULL; + } + + return v; +} + +char *PyStructSequence_UnnamedField = "unnamed field"; + +PyDoc_STRVAR(stat_result__doc__, +"stat_result: Result from stat, fstat, or lstat.\n\n\ +This object may be accessed either as a tuple of\n\ + (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)\n\ +or via the attributes st_mode, st_ino, st_dev, st_nlink, st_uid, and so on.\n\ +\n\ +Posix/windows: If your platform supports st_blksize, st_blocks, st_rdev,\n\ +or st_flags, they are available as attributes only.\n\ +\n\ +See os.stat for more information."); + +static PyStructSequence_Field stat_result_fields[] = { + {"st_mode", "protection bits"}, + {"st_ino", "inode"}, + {"st_dev", "device"}, + {"st_nlink", "number of hard links"}, + {"st_uid", "user ID of owner"}, + {"st_gid", "group ID of owner"}, + {"st_size", "total size, in bytes"}, + /* The NULL is replaced with PyStructSequence_UnnamedField later. */ + {NULL, "integer time of last access"}, + {NULL, "integer time of last modification"}, + {NULL, "integer time of last change"}, + {"st_atime", "time of last access"}, + {"st_mtime", "time of last modification"}, + {"st_ctime", "time of last change"}, + {"st_atime_ns", "time of last access in nanoseconds"}, + {"st_mtime_ns", "time of last modification in nanoseconds"}, + {"st_ctime_ns", "time of last change in nanoseconds"}, +#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE + {"st_blksize", "blocksize for filesystem I/O"}, +#endif +#ifdef HAVE_STRUCT_STAT_ST_BLOCKS + {"st_blocks", "number of blocks allocated"}, +#endif +#ifdef HAVE_STRUCT_STAT_ST_RDEV + {"st_rdev", "device type (if inode device)"}, +#endif +#ifdef HAVE_STRUCT_STAT_ST_FLAGS + {"st_flags", "user defined flags for file"}, +#endif +#ifdef HAVE_STRUCT_STAT_ST_GEN + {"st_gen", "generation number"}, +#endif +#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME + {"st_birthtime", "time of creation"}, +#endif +#ifdef HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES + {"st_file_attributes", "Windows file attribute bits"}, +#endif + {0} +}; + +static PyStructSequence_Desc stat_result_desc = { + "scandir.stat_result", /* name */ + stat_result__doc__, /* doc */ + stat_result_fields, + 10 +}; + + +#ifdef MS_WINDOWS +static int +win32_warn_bytes_api() +{ + return PyErr_WarnEx(PyExc_DeprecationWarning, + "The Windows bytes API has been deprecated, " + "use Unicode filenames instead", + 1); +} +#endif + +typedef struct { + const char *function_name; + const char *argument_name; + int nullable; + wchar_t *wide; + char *narrow; + int fd; + Py_ssize_t length; + PyObject *object; + PyObject *cleanup; +} path_t; + +static void +path_cleanup(path_t *path) { + if (path->cleanup) { + Py_CLEAR(path->cleanup); + } +} + +static int +path_converter(PyObject *o, void *p) { + path_t *path = (path_t *)p; + PyObject *unicode, *bytes; + Py_ssize_t length; + char *narrow; + +#define FORMAT_EXCEPTION(exc, fmt) \ + PyErr_Format(exc, "%s%s" fmt, \ + path->function_name ? path->function_name : "", \ + path->function_name ? ": " : "", \ + path->argument_name ? path->argument_name : "path") + + /* Py_CLEANUP_SUPPORTED support */ + if (o == NULL) { + path_cleanup(path); + return 1; + } + + /* ensure it's always safe to call path_cleanup() */ + path->cleanup = NULL; + + if (o == Py_None) { + if (!path->nullable) { + FORMAT_EXCEPTION(PyExc_TypeError, + "can't specify None for %s argument"); + return 0; + } + path->wide = NULL; + path->narrow = NULL; + path->length = 0; + path->object = o; + path->fd = -1; + return 1; + } + + unicode = PyUnicode_FromObject(o); + if (unicode) { +#ifdef MS_WINDOWS + wchar_t *wide; + + wide = PyUnicode_AsUnicodeAndSize(unicode, &length); + if (!wide) { + Py_DECREF(unicode); + return 0; + } + if (length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + Py_DECREF(unicode); + return 0; + } + if (wcslen(wide) != length) { + FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character"); + Py_DECREF(unicode); + return 0; + } + + path->wide = wide; + path->narrow = NULL; + path->length = length; + path->object = o; + path->fd = -1; + path->cleanup = unicode; + return Py_CLEANUP_SUPPORTED; +#else +#if PY_MAJOR_VERSION >= 3 + if (!PyUnicode_FSConverter(unicode, &bytes)) + bytes = NULL; +#else + bytes = PyUnicode_AsEncodedString(unicode, FS_ENCODING, "strict"); +#endif + Py_DECREF(unicode); +#endif + } + else { + PyErr_Clear(); +#if PY_MAJOR_VERSION >= 3 + if (PyObject_CheckBuffer(o)) { + bytes = PyBytes_FromObject(o); + } +#else + if (PyString_Check(o)) { + bytes = o; + Py_INCREF(bytes); + } +#endif + else + bytes = NULL; + if (!bytes) { + PyErr_Clear(); + } + } + + if (!bytes) { + if (!PyErr_Occurred()) + FORMAT_EXCEPTION(PyExc_TypeError, "illegal type for %s parameter"); + return 0; + } + +#ifdef MS_WINDOWS + if (win32_warn_bytes_api()) { + Py_DECREF(bytes); + return 0; + } +#endif + + length = PyBytes_GET_SIZE(bytes); +#ifdef MS_WINDOWS + if (length > MAX_PATH-1) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + Py_DECREF(bytes); + return 0; + } +#endif + + narrow = PyBytes_AS_STRING(bytes); + if ((size_t)length != strlen(narrow)) { + FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); + Py_DECREF(bytes); + return 0; + } + + path->wide = NULL; + path->narrow = narrow; + path->length = length; + path->object = o; + path->fd = -1; + path->cleanup = bytes; + return Py_CLEANUP_SUPPORTED; +} + +static PyObject * +path_error(path_t *path) +{ +#ifdef MS_WINDOWS + return PyErr_SetExcFromWindowsErrWithFilenameObject(PyExc_OSError, + 0, path->object); +#else + return PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path->object); +#endif +} + + +/* SECTION: Main DirEntry and scandir implementation, taken from + Python 3.5's posixmodule.c */ + +PyDoc_STRVAR(posix_scandir__doc__, +"scandir(path='.') -> iterator of DirEntry objects for given path"); + +static char *follow_symlinks_keywords[] = {"follow_symlinks", NULL}; +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 +static char *follow_symlinks_format = "|$p:DirEntry.stat"; +#else +static char *follow_symlinks_format = "|i:DirEntry.stat"; +#endif + +typedef struct { + PyObject_HEAD + PyObject *name; + PyObject *path; + PyObject *stat; + PyObject *lstat; +#ifdef MS_WINDOWS + struct _Py_stat_struct win32_lstat; + unsigned __int64 win32_file_index; + int got_file_index; +#if PY_MAJOR_VERSION < 3 + int name_path_bytes; +#endif +#else /* POSIX */ +#ifdef HAVE_DIRENT_D_TYPE + unsigned char d_type; +#endif + ino_t d_ino; +#endif +} DirEntry; + +static void +DirEntry_dealloc(DirEntry *entry) +{ + Py_XDECREF(entry->name); + Py_XDECREF(entry->path); + Py_XDECREF(entry->stat); + Py_XDECREF(entry->lstat); + Py_TYPE(entry)->tp_free((PyObject *)entry); +} + +/* Forward reference */ +static int +DirEntry_test_mode(DirEntry *self, int follow_symlinks, unsigned short mode_bits); + +/* Set exception and return -1 on error, 0 for False, 1 for True */ +static int +DirEntry_is_symlink(DirEntry *self) +{ +#ifdef MS_WINDOWS + return (self->win32_lstat.st_mode & S_IFMT) == S_IFLNK; +#elif defined(HAVE_DIRENT_D_TYPE) + /* POSIX */ + if (self->d_type != DT_UNKNOWN) + return self->d_type == DT_LNK; + else + return DirEntry_test_mode(self, 0, S_IFLNK); +#else + /* POSIX without d_type */ + return DirEntry_test_mode(self, 0, S_IFLNK); +#endif +} + +static PyObject * +DirEntry_py_is_symlink(DirEntry *self) +{ + int result; + + result = DirEntry_is_symlink(self); + if (result == -1) + return NULL; + return PyBool_FromLong(result); +} + +static PyObject * +DirEntry_fetch_stat(DirEntry *self, int follow_symlinks) +{ + int result; + struct _Py_stat_struct st; + +#ifdef MS_WINDOWS + wchar_t *path; + + path = PyUnicode_AsUnicode(self->path); + if (!path) + return NULL; + + if (follow_symlinks) + result = win32_stat_w(path, &st); + else + result = win32_lstat_w(path, &st); + + if (result != 0) { + return PyErr_SetExcFromWindowsErrWithFilenameObject(PyExc_OSError, + 0, self->path); + } +#else /* POSIX */ + PyObject *bytes; + char *path; + +#if PY_MAJOR_VERSION >= 3 + if (!PyUnicode_FSConverter(self->path, &bytes)) + return NULL; +#else + if (PyString_Check(self->path)) { + bytes = self->path; + Py_INCREF(bytes); + } else { + bytes = PyUnicode_AsEncodedString(self->path, FS_ENCODING, "strict"); + if (!bytes) + return NULL; + } +#endif + path = PyBytes_AS_STRING(bytes); + + if (follow_symlinks) + result = STAT(path, &st); + else + result = LSTAT(path, &st); + Py_DECREF(bytes); + + if (result != 0) + return PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, self->path); +#endif + + return _pystat_fromstructstat(&st); +} + +static PyObject * +DirEntry_get_lstat(DirEntry *self) +{ + if (!self->lstat) { +#ifdef MS_WINDOWS + self->lstat = _pystat_fromstructstat(&self->win32_lstat); +#else /* POSIX */ + self->lstat = DirEntry_fetch_stat(self, 0); +#endif + } + Py_XINCREF(self->lstat); + return self->lstat; +} + +static PyObject * +DirEntry_get_stat(DirEntry *self, int follow_symlinks) +{ + if (!follow_symlinks) + return DirEntry_get_lstat(self); + + if (!self->stat) { + int result = DirEntry_is_symlink(self); + if (result == -1) + return NULL; + else if (result) + self->stat = DirEntry_fetch_stat(self, 1); + else + self->stat = DirEntry_get_lstat(self); + } + + Py_XINCREF(self->stat); + return self->stat; +} + +static PyObject * +DirEntry_stat(DirEntry *self, PyObject *args, PyObject *kwargs) +{ + int follow_symlinks = 1; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, follow_symlinks_format, + follow_symlinks_keywords, &follow_symlinks)) + return NULL; + + return DirEntry_get_stat(self, follow_symlinks); +} + +/* Set exception and return -1 on error, 0 for False, 1 for True */ +static int +DirEntry_test_mode(DirEntry *self, int follow_symlinks, unsigned short mode_bits) +{ + PyObject *stat = NULL; + PyObject *st_mode = NULL; + long mode; + int result; +#if defined(MS_WINDOWS) || defined(HAVE_DIRENT_D_TYPE) + int is_symlink; + int need_stat; +#endif +#ifdef MS_WINDOWS + unsigned long dir_bits; +#endif + _Py_IDENTIFIER(st_mode); + +#ifdef MS_WINDOWS + is_symlink = (self->win32_lstat.st_mode & S_IFMT) == S_IFLNK; + need_stat = follow_symlinks && is_symlink; +#elif defined(HAVE_DIRENT_D_TYPE) + is_symlink = self->d_type == DT_LNK; + need_stat = self->d_type == DT_UNKNOWN || (follow_symlinks && is_symlink); +#endif + +#if defined(MS_WINDOWS) || defined(HAVE_DIRENT_D_TYPE) + if (need_stat) { +#endif + stat = DirEntry_get_stat(self, follow_symlinks); + if (!stat) { + if (PyErr_ExceptionMatches(PyExc_FileNotFoundError)) { + /* If file doesn't exist (anymore), then return False + (i.e., say it's not a file/directory) */ + PyErr_Clear(); + return 0; + } + goto error; + } + st_mode = _PyObject_GetAttrId(stat, &PyId_st_mode); + if (!st_mode) + goto error; + + mode = PyLong_AsLong(st_mode); + if (mode == -1 && PyErr_Occurred()) + goto error; + Py_CLEAR(st_mode); + Py_CLEAR(stat); + result = (mode & S_IFMT) == mode_bits; +#if defined(MS_WINDOWS) || defined(HAVE_DIRENT_D_TYPE) + } + else if (is_symlink) { + assert(mode_bits != S_IFLNK); + result = 0; + } + else { + assert(mode_bits == S_IFDIR || mode_bits == S_IFREG); +#ifdef MS_WINDOWS + dir_bits = self->win32_lstat.st_file_attributes & FILE_ATTRIBUTE_DIRECTORY; + if (mode_bits == S_IFDIR) + result = dir_bits != 0; + else + result = dir_bits == 0; +#else /* POSIX */ + if (mode_bits == S_IFDIR) + result = self->d_type == DT_DIR; + else + result = self->d_type == DT_REG; +#endif + } +#endif + + return result; + +error: + Py_XDECREF(st_mode); + Py_XDECREF(stat); + return -1; +} + +static PyObject * +DirEntry_py_test_mode(DirEntry *self, int follow_symlinks, unsigned short mode_bits) +{ + int result; + + result = DirEntry_test_mode(self, follow_symlinks, mode_bits); + if (result == -1) + return NULL; + return PyBool_FromLong(result); +} + +static PyObject * +DirEntry_is_dir(DirEntry *self, PyObject *args, PyObject *kwargs) +{ + int follow_symlinks = 1; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, follow_symlinks_format, + follow_symlinks_keywords, &follow_symlinks)) + return NULL; + + return DirEntry_py_test_mode(self, follow_symlinks, S_IFDIR); +} + +static PyObject * +DirEntry_is_file(DirEntry *self, PyObject *args, PyObject *kwargs) +{ + int follow_symlinks = 1; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, follow_symlinks_format, + follow_symlinks_keywords, &follow_symlinks)) + return NULL; + + return DirEntry_py_test_mode(self, follow_symlinks, S_IFREG); +} + +static PyObject * +DirEntry_inode(DirEntry *self) +{ +#ifdef MS_WINDOWS + if (!self->got_file_index) { + wchar_t *path; + struct _Py_stat_struct stat; + + path = PyUnicode_AsUnicode(self->path); + if (!path) + return NULL; + + if (win32_lstat_w(path, &stat) != 0) { + return PyErr_SetExcFromWindowsErrWithFilenameObject(PyExc_OSError, + 0, self->path); + } + + self->win32_file_index = stat.st_ino; + self->got_file_index = 1; + } + return PyLong_FromUnsignedLongLong(self->win32_file_index); +#else /* POSIX */ +#ifdef HAVE_LARGEFILE_SUPPORT + return PyLong_FromUnsignedLongLong(self->d_ino); +#else + return PyLong_FromUnsignedLong((unsigned long)self->d_ino); +#endif +#endif +} + +#if PY_MAJOR_VERSION < 3 && defined(MS_WINDOWS) + +PyObject *DirEntry_name_getter(DirEntry *self, void *closure) { + if (self->name_path_bytes) { + return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(self->name), + PyUnicode_GetSize(self->name), "strict"); + } else { + Py_INCREF(self->name); + return self->name; + } +} + +PyObject *DirEntry_path_getter(DirEntry *self, void *closure) { + if (self->name_path_bytes) { + return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(self->path), + PyUnicode_GetSize(self->path), "strict"); + } else { + Py_INCREF(self->path); + return self->path; + } +} + +static PyGetSetDef DirEntry_getset[] = { + {"name", (getter)DirEntry_name_getter, NULL, + "the entry's base filename, relative to scandir() \"path\" argument", NULL}, + {"path", (getter)DirEntry_path_getter, NULL, + "the entry's full path name; equivalent to os.path.join(scandir_path, entry.name)", NULL}, + {NULL} +}; + +#else + +static PyMemberDef DirEntry_members[] = { + {"name", T_OBJECT_EX, offsetof(DirEntry, name), READONLY, + "the entry's base filename, relative to scandir() \"path\" argument"}, + {"path", T_OBJECT_EX, offsetof(DirEntry, path), READONLY, + "the entry's full path name; equivalent to os.path.join(scandir_path, entry.name)"}, + {NULL} +}; + +#endif + +static PyObject * +DirEntry_repr(DirEntry *self) +{ +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromFormat("<DirEntry %R>", self->name); +#elif defined(MS_WINDOWS) + PyObject *name; + PyObject *name_repr; + PyObject *entry_repr; + + name = DirEntry_name_getter(self, NULL); + if (!name) + return NULL; + name_repr = PyObject_Repr(name); + Py_DECREF(name); + if (!name_repr) + return NULL; + entry_repr = PyString_FromFormat("<DirEntry %s>", PyString_AsString(name_repr)); + Py_DECREF(name_repr); + return entry_repr; +#else + PyObject *name_repr; + PyObject *entry_repr; + + name_repr = PyObject_Repr(self->name); + if (!name_repr) + return NULL; + entry_repr = PyString_FromFormat("<DirEntry %s>", PyString_AsString(name_repr)); + Py_DECREF(name_repr); + return entry_repr; +#endif +} + +static PyMethodDef DirEntry_methods[] = { + {"is_dir", (PyCFunction)DirEntry_is_dir, METH_VARARGS | METH_KEYWORDS, + "return True if the entry is a directory; cached per entry" + }, + {"is_file", (PyCFunction)DirEntry_is_file, METH_VARARGS | METH_KEYWORDS, + "return True if the entry is a file; cached per entry" + }, + {"is_symlink", (PyCFunction)DirEntry_py_is_symlink, METH_NOARGS, + "return True if the entry is a symbolic link; cached per entry" + }, + {"stat", (PyCFunction)DirEntry_stat, METH_VARARGS | METH_KEYWORDS, + "return stat_result object for the entry; cached per entry" + }, + {"inode", (PyCFunction)DirEntry_inode, METH_NOARGS, + "return inode of the entry; cached per entry", + }, + {NULL} +}; + +static PyTypeObject DirEntryType = { + PyVarObject_HEAD_INIT(NULL, 0) + MODNAME ".DirEntry", /* tp_name */ + sizeof(DirEntry), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)DirEntry_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)DirEntry_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + DirEntry_methods, /* tp_methods */ +#if PY_MAJOR_VERSION < 3 && defined(MS_WINDOWS) + NULL, /* tp_members */ + DirEntry_getset, /* tp_getset */ +#else + DirEntry_members, /* tp_members */ + NULL, /* tp_getset */ +#endif +}; + +#ifdef MS_WINDOWS + +static wchar_t * +join_path_filenameW(wchar_t *path_wide, wchar_t* filename) +{ + Py_ssize_t path_len; + Py_ssize_t size; + wchar_t *result; + wchar_t ch; + + if (!path_wide) { /* Default arg: "." */ + path_wide = L"."; + path_len = 1; + } + else { + path_len = wcslen(path_wide); + } + + /* The +1's are for the path separator and the NUL */ + size = path_len + 1 + wcslen(filename) + 1; + result = PyMem_New(wchar_t, size); + if (!result) { + PyErr_NoMemory(); + return NULL; + } + wcscpy(result, path_wide); + if (path_len > 0) { + ch = result[path_len - 1]; + if (ch != SEP && ch != ALTSEP && ch != L':') + result[path_len++] = SEP; + wcscpy(result + path_len, filename); + } + return result; +} + +static PyObject * +DirEntry_from_find_data(path_t *path, WIN32_FIND_DATAW *dataW) +{ + DirEntry *entry; + BY_HANDLE_FILE_INFORMATION file_info; + ULONG reparse_tag; + wchar_t *joined_path; + + entry = PyObject_New(DirEntry, &DirEntryType); + if (!entry) + return NULL; + entry->name = NULL; + entry->path = NULL; + entry->stat = NULL; + entry->lstat = NULL; + entry->got_file_index = 0; +#if PY_MAJOR_VERSION < 3 + entry->name_path_bytes = path->object && PyBytes_Check(path->object); +#endif + + entry->name = PyUnicode_FromWideChar(dataW->cFileName, wcslen(dataW->cFileName)); + if (!entry->name) + goto error; + + joined_path = join_path_filenameW(path->wide, dataW->cFileName); + if (!joined_path) + goto error; + + entry->path = PyUnicode_FromWideChar(joined_path, wcslen(joined_path)); + PyMem_Free(joined_path); + if (!entry->path) + goto error; + + find_data_to_file_info_w(dataW, &file_info, &reparse_tag); + _Py_attribute_data_to_stat(&file_info, reparse_tag, &entry->win32_lstat); + + return (PyObject *)entry; + +error: + Py_DECREF(entry); + return NULL; +} + +#else /* POSIX */ + +static char * +join_path_filename(char *path_narrow, char* filename, Py_ssize_t filename_len) +{ + Py_ssize_t path_len; + Py_ssize_t size; + char *result; + + if (!path_narrow) { /* Default arg: "." */ + path_narrow = "."; + path_len = 1; + } + else { + path_len = strlen(path_narrow); + } + + if (filename_len == -1) + filename_len = strlen(filename); + + /* The +1's are for the path separator and the NUL */ + size = path_len + 1 + filename_len + 1; + result = PyMem_New(char, size); + if (!result) { + PyErr_NoMemory(); + return NULL; + } + strcpy(result, path_narrow); + if (path_len > 0 && result[path_len - 1] != '/') + result[path_len++] = '/'; + strcpy(result + path_len, filename); + return result; +} + +static PyObject * +DirEntry_from_posix_info(path_t *path, char *name, Py_ssize_t name_len, + ino_t d_ino +#ifdef HAVE_DIRENT_D_TYPE + , unsigned char d_type +#endif + ) +{ + DirEntry *entry; + char *joined_path; + + entry = PyObject_New(DirEntry, &DirEntryType); + if (!entry) + return NULL; + entry->name = NULL; + entry->path = NULL; + entry->stat = NULL; + entry->lstat = NULL; + + joined_path = join_path_filename(path->narrow, name, name_len); + if (!joined_path) + goto error; + + if (!path->narrow || !PyBytes_Check(path->object)) { +#if PY_MAJOR_VERSION >= 3 + entry->name = PyUnicode_DecodeFSDefaultAndSize(name, name_len); + entry->path = PyUnicode_DecodeFSDefault(joined_path); +#else + entry->name = PyUnicode_Decode(name, name_len, + FS_ENCODING, "strict"); + entry->path = PyUnicode_Decode(joined_path, strlen(joined_path), + FS_ENCODING, "strict"); +#endif + } + else { + entry->name = PyBytes_FromStringAndSize(name, name_len); + entry->path = PyBytes_FromString(joined_path); + } + PyMem_Free(joined_path); + if (!entry->name || !entry->path) + goto error; + +#ifdef HAVE_DIRENT_D_TYPE + entry->d_type = d_type; +#endif + entry->d_ino = d_ino; + + return (PyObject *)entry; + +error: + Py_XDECREF(entry); + return NULL; +} + +#endif + + +typedef struct { + PyObject_HEAD + path_t path; +#ifdef MS_WINDOWS + HANDLE handle; + WIN32_FIND_DATAW file_data; + int first_time; +#else /* POSIX */ + DIR *dirp; +#endif +} ScandirIterator; + +#ifdef MS_WINDOWS + +static void +ScandirIterator_close(ScandirIterator *iterator) +{ + if (iterator->handle == INVALID_HANDLE_VALUE) + return; + + Py_BEGIN_ALLOW_THREADS + FindClose(iterator->handle); + Py_END_ALLOW_THREADS + iterator->handle = INVALID_HANDLE_VALUE; +} + +static PyObject * +ScandirIterator_iternext(ScandirIterator *iterator) +{ + WIN32_FIND_DATAW *file_data = &iterator->file_data; + BOOL success; + + /* Happens if the iterator is iterated twice */ + if (iterator->handle == INVALID_HANDLE_VALUE) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + while (1) { + if (!iterator->first_time) { + Py_BEGIN_ALLOW_THREADS + success = FindNextFileW(iterator->handle, file_data); + Py_END_ALLOW_THREADS + if (!success) { + if (GetLastError() != ERROR_NO_MORE_FILES) + return path_error(&iterator->path); + /* No more files found in directory, stop iterating */ + break; + } + } + iterator->first_time = 0; + + /* Skip over . and .. */ + if (wcscmp(file_data->cFileName, L".") != 0 && + wcscmp(file_data->cFileName, L"..") != 0) + return DirEntry_from_find_data(&iterator->path, file_data); + + /* Loop till we get a non-dot directory or finish iterating */ + } + + ScandirIterator_close(iterator); + + PyErr_SetNone(PyExc_StopIteration); + return NULL; +} + +#else /* POSIX */ + +static void +ScandirIterator_close(ScandirIterator *iterator) +{ + if (!iterator->dirp) + return; + + Py_BEGIN_ALLOW_THREADS + closedir(iterator->dirp); + Py_END_ALLOW_THREADS + iterator->dirp = NULL; + return; +} + +static PyObject * +ScandirIterator_iternext(ScandirIterator *iterator) +{ + struct dirent *direntp; + Py_ssize_t name_len; + int is_dot; + + /* Happens if the iterator is iterated twice */ + if (!iterator->dirp) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + while (1) { + errno = 0; + Py_BEGIN_ALLOW_THREADS + direntp = readdir(iterator->dirp); + Py_END_ALLOW_THREADS + + if (!direntp) { + if (errno != 0) + return path_error(&iterator->path); + /* No more files found in directory, stop iterating */ + break; + } + + /* Skip over . and .. */ + name_len = NAMLEN(direntp); + is_dot = direntp->d_name[0] == '.' && + (name_len == 1 || (direntp->d_name[1] == '.' && name_len == 2)); + if (!is_dot) { + return DirEntry_from_posix_info(&iterator->path, direntp->d_name, + name_len, direntp->d_ino +#ifdef HAVE_DIRENT_D_TYPE + , direntp->d_type +#endif + ); + } + + /* Loop till we get a non-dot directory or finish iterating */ + } + + ScandirIterator_close(iterator); + + PyErr_SetNone(PyExc_StopIteration); + return NULL; +} + +#endif + +static void +ScandirIterator_dealloc(ScandirIterator *iterator) +{ + ScandirIterator_close(iterator); + Py_XDECREF(iterator->path.object); + path_cleanup(&iterator->path); + Py_TYPE(iterator)->tp_free((PyObject *)iterator); +} + +static PyTypeObject ScandirIteratorType = { + PyVarObject_HEAD_INIT(NULL, 0) + MODNAME ".ScandirIterator", /* tp_name */ + sizeof(ScandirIterator), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)ScandirIterator_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)ScandirIterator_iternext, /* tp_iternext */ +}; + +static PyObject * +posix_scandir(PyObject *self, PyObject *args, PyObject *kwargs) +{ + ScandirIterator *iterator; + static char *keywords[] = {"path", NULL}; +#ifdef MS_WINDOWS + wchar_t *path_strW; +#else + char *path; +#endif + + iterator = PyObject_New(ScandirIterator, &ScandirIteratorType); + if (!iterator) + return NULL; + memset(&iterator->path, 0, sizeof(path_t)); + iterator->path.function_name = "scandir"; + iterator->path.nullable = 1; + +#ifdef MS_WINDOWS + iterator->handle = INVALID_HANDLE_VALUE; +#else + iterator->dirp = NULL; +#endif + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O&:scandir", keywords, + path_converter, &iterator->path)) + goto error; + + /* path_converter doesn't keep path.object around, so do it + manually for the lifetime of the iterator here (the refcount + is decremented in ScandirIterator_dealloc) + */ + Py_XINCREF(iterator->path.object); + +#ifdef MS_WINDOWS + if (iterator->path.narrow) { + PyErr_SetString(PyExc_TypeError, + "os.scandir() doesn't support bytes path on Windows, use Unicode instead"); + goto error; + } + iterator->first_time = 1; + + path_strW = join_path_filenameW(iterator->path.wide, L"*.*"); + if (!path_strW) + goto error; + + Py_BEGIN_ALLOW_THREADS + iterator->handle = FindFirstFileW(path_strW, &iterator->file_data); + Py_END_ALLOW_THREADS + + PyMem_Free(path_strW); + + if (iterator->handle == INVALID_HANDLE_VALUE) { + path_error(&iterator->path); + goto error; + } +#else /* POSIX */ + if (iterator->path.narrow) + path = iterator->path.narrow; + else + path = "."; + + errno = 0; + Py_BEGIN_ALLOW_THREADS + iterator->dirp = opendir(path); + Py_END_ALLOW_THREADS + + if (!iterator->dirp) { + path_error(&iterator->path); + goto error; + } +#endif + + return (PyObject *)iterator; + +error: + Py_DECREF(iterator); + return NULL; +} + + +/* SECTION: Module and method definitions and initialization code */ + +static PyMethodDef scandir_methods[] = { + {"scandir", (PyCFunction)posix_scandir, + METH_VARARGS | METH_KEYWORDS, + posix_scandir__doc__}, + {NULL, NULL}, +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_scandir", + NULL, + 0, + scandir_methods, + NULL, + NULL, + NULL, + NULL, +}; +#endif + +#if PY_MAJOR_VERSION >= 3 +PyObject * +PyInit__scandir(void) +{ + PyObject *module = PyModule_Create(&moduledef); +#else +void +init_scandir(void) +{ + PyObject *module = Py_InitModule("_scandir", scandir_methods); +#endif + if (module == NULL) { + INIT_ERROR; + } + + billion = PyLong_FromLong(1000000000); + if (!billion) + INIT_ERROR; + + stat_result_desc.fields[7].name = PyStructSequence_UnnamedField; + stat_result_desc.fields[8].name = PyStructSequence_UnnamedField; + stat_result_desc.fields[9].name = PyStructSequence_UnnamedField; + PyStructSequence_InitType(&StatResultType, &stat_result_desc); + structseq_new = StatResultType.tp_new; + StatResultType.tp_new = statresult_new; + + if (PyType_Ready(&ScandirIteratorType) < 0) + INIT_ERROR; + if (PyType_Ready(&DirEntryType) < 0) + INIT_ERROR; + +#if PY_MAJOR_VERSION >= 3 + return module; +#endif +} diff --git a/Contents/Libraries/Shared/anydbm.py b/Contents/Libraries/Shared/anydbm.py new file mode 100644 index 000000000..ba7e90510 --- /dev/null +++ b/Contents/Libraries/Shared/anydbm.py @@ -0,0 +1,85 @@ +"""Generic interface to all dbm clones. + +Instead of + + import dbm + d = dbm.open(file, 'w', 0666) + +use + + import anydbm + d = anydbm.open(file, 'w') + +The returned object is a dbhash, gdbm, dbm or dumbdbm object, +dependent on the type of database being opened (determined by whichdb +module) in the case of an existing dbm. If the dbm does not exist and +the create or new flag ('c' or 'n') was specified, the dbm type will +be determined by the availability of the modules (tested in the above +order). + +It has the following interface (key and data are strings): + + d[key] = data # store data at key (may override data at + # existing key) + data = d[key] # retrieve data at key (raise KeyError if no + # such key) + del d[key] # delete data stored at key (raises KeyError + # if no such key) + flag = key in d # true if the key exists + list = d.keys() # return a list of all existing keys (slow!) + +Future versions may change the order in which implementations are +tested for existence, and add interfaces to other dbm-like +implementations. +""" + +class error(Exception): + pass + +_names = ['dbhash', 'gdbm', 'dbm', 'dumbdbm'] +_errors = [error] +_defaultmod = None + +for _name in _names: + try: + _mod = __import__(_name) + except ImportError: + continue + if not _defaultmod: + _defaultmod = _mod + _errors.append(_mod.error) + +if not _defaultmod: + raise ImportError, "no dbm clone found; tried %s" % _names + +error = tuple(_errors) + +def open(file, flag='r', mode=0666): + """Open or create database at path given by *file*. + + Optional argument *flag* can be 'r' (default) for read-only access, 'w' + for read-write access of an existing database, 'c' for read-write access + to a new or existing database, and 'n' for read-write access to a new + database. + + Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it + only if it doesn't exist; and 'n' always creates a new database. + """ + + # guess the type of an existing database + from whichdb import whichdb + result=whichdb(file) + if result is None: + # db doesn't exist + if 'c' in flag or 'n' in flag: + # file doesn't exist and the new + # flag was used so use default type + mod = _defaultmod + else: + raise error, "need 'c' or 'n' flag to open new db" + elif result == "": + # db type cannot be determined + raise error, "db type could not be determined" + else: + mod = __import__(result) + return mod.open(file, flag, mode) diff --git a/Contents/Libraries/Shared/appdirs.py b/Contents/Libraries/Shared/appdirs.py new file mode 100644 index 000000000..f4dba0953 --- /dev/null +++ b/Contents/Libraries/Shared/appdirs.py @@ -0,0 +1,552 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2005-2010 ActiveState Software Inc. +# Copyright (c) 2013 Eddy Petrișor + +"""Utilities for determining application-specific dirs. + +See <http://github.com/ActiveState/appdirs> for details and usage. +""" +# Dev Notes: +# - MSDN on where to store app data files: +# http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120 +# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html +# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html + +__version_info__ = (1, 4, 0) +__version__ = '.'.join(map(str, __version_info__)) + + +import sys +import os + +PY3 = sys.version_info[0] == 3 + +if PY3: + unicode = str + +if sys.platform.startswith('java'): + import platform + os_name = platform.java_ver()[3][0] + if os_name.startswith('Windows'): # "Windows XP", "Windows 7", etc. + system = 'win32' + elif os_name.startswith('Mac'): # "Mac OS X", etc. + system = 'darwin' + else: # "Linux", "SunOS", "FreeBSD", etc. + # Setting this to "linux2" is not ideal, but only Windows or Mac + # are actually checked for and the rest of the module expects + # *sys.platform* style strings. + system = 'linux2' +else: + system = sys.platform + + + +def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> + for a discussion of issues. + + Typical user data directories are: + Mac OS X: ~/Library/Application Support/<AppName> + Unix: ~/.local/share/<AppName> # or in $XDG_DATA_HOME, if defined + Win XP (not roaming): C:\Documents and Settings\<username>\Application Data\<AppAuthor>\<AppName> + Win XP (roaming): C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName> + Win 7 (not roaming): C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName> + Win 7 (roaming): C:\Users\<username>\AppData\Roaming\<AppAuthor>\<AppName> + + For Unix, we follow the XDG spec and support $XDG_DATA_HOME. + That means, by default "~/.local/share/<AppName>". + """ + if system == "win32": + if appauthor is None: + appauthor = appname + const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA" + path = os.path.normpath(_get_win_folder(const)) + if appname: + if appauthor is not False: + path = os.path.join(path, appauthor, appname) + else: + path = os.path.join(path, appname) + elif system == 'darwin': + path = os.path.expanduser('~/Library/Application Support/') + if appname: + path = os.path.join(path, appname) + else: + path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): + """Return full path to the user-shared data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "multipath" is an optional parameter only applicable to *nix + which indicates that the entire list of data dirs should be + returned. By default, the first item from XDG_DATA_DIRS is + returned, or '/usr/local/share/<AppName>', + if XDG_DATA_DIRS is not set + + Typical user data directories are: + Mac OS X: /Library/Application Support/<AppName> + Unix: /usr/local/share/<AppName> or /usr/share/<AppName> + Win XP: C:\Documents and Settings\All Users\Application Data\<AppAuthor>\<AppName> + Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) + Win 7: C:\ProgramData\<AppAuthor>\<AppName> # Hidden, but writeable on Win 7. + + For Unix, this is using the $XDG_DATA_DIRS[0] default. + + WARNING: Do not use this on Windows. See the Vista-Fail note above for why. + """ + if system == "win32": + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA")) + if appname: + if appauthor is not False: + path = os.path.join(path, appauthor, appname) + else: + path = os.path.join(path, appname) + elif system == 'darwin': + path = os.path.expanduser('/Library/Application Support') + if appname: + path = os.path.join(path, appname) + else: + # XDG default for $XDG_DATA_DIRS + # only first, if multipath is False + path = os.getenv('XDG_DATA_DIRS', + os.pathsep.join(['/usr/local/share', '/usr/share'])) + pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] + if appname: + if version: + appname = os.path.join(appname, version) + pathlist = [os.sep.join([x, appname]) for x in pathlist] + + if multipath: + path = os.pathsep.join(pathlist) + else: + path = pathlist[0] + return path + + if appname and version: + path = os.path.join(path, version) + return path + + +def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific config dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> + for a discussion of issues. + + Typical user data directories are: + Mac OS X: same as user_data_dir + Unix: ~/.config/<AppName> # or in $XDG_CONFIG_HOME, if defined + Win *: same as user_data_dir + + For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME. + That means, by deafult "~/.config/<AppName>". + """ + if system in ["win32", "darwin"]: + path = user_data_dir(appname, appauthor, None, roaming) + else: + path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): + """Return full path to the user-shared data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "multipath" is an optional parameter only applicable to *nix + which indicates that the entire list of config dirs should be + returned. By default, the first item from XDG_CONFIG_DIRS is + returned, or '/etc/xdg/<AppName>', if XDG_CONFIG_DIRS is not set + + Typical user data directories are: + Mac OS X: same as site_data_dir + Unix: /etc/xdg/<AppName> or $XDG_CONFIG_DIRS[i]/<AppName> for each value in + $XDG_CONFIG_DIRS + Win *: same as site_data_dir + Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) + + For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False + + WARNING: Do not use this on Windows. See the Vista-Fail note above for why. + """ + if system in ["win32", "darwin"]: + path = site_data_dir(appname, appauthor) + if appname and version: + path = os.path.join(path, version) + else: + # XDG default for $XDG_CONFIG_DIRS + # only first, if multipath is False + path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') + pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] + if appname: + if version: + appname = os.path.join(appname, version) + pathlist = [os.sep.join([x, appname]) for x in pathlist] + + if multipath: + path = os.pathsep.join(pathlist) + else: + path = pathlist[0] + return path + + +def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): + r"""Return full path to the user-specific cache dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "opinion" (boolean) can be False to disable the appending of + "Cache" to the base app data dir for Windows. See + discussion below. + + Typical user cache directories are: + Mac OS X: ~/Library/Caches/<AppName> + Unix: ~/.cache/<AppName> (XDG default) + Win XP: C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Cache + Vista: C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Cache + + On Windows the only suggestion in the MSDN docs is that local settings go in + the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming + app data dir (the default returned by `user_data_dir` above). Apps typically + put cache data somewhere *under* the given dir here. Some examples: + ...\Mozilla\Firefox\Profiles\<ProfileName>\Cache + ...\Acme\SuperApp\Cache\1.0 + OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value. + This can be disabled with the `opinion=False` option. + """ + if system == "win32": + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) + if appname: + if appauthor is not False: + path = os.path.join(path, appauthor, appname) + else: + path = os.path.join(path, appname) + if opinion: + path = os.path.join(path, "Cache") + elif system == 'darwin': + path = os.path.expanduser('~/Library/Caches') + if appname: + path = os.path.join(path, appname) + else: + path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def user_log_dir(appname=None, appauthor=None, version=None, opinion=True): + r"""Return full path to the user-specific log dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "opinion" (boolean) can be False to disable the appending of + "Logs" to the base app data dir for Windows, and "log" to the + base cache dir for Unix. See discussion below. + + Typical user cache directories are: + Mac OS X: ~/Library/Logs/<AppName> + Unix: ~/.cache/<AppName>/log # or under $XDG_CACHE_HOME if defined + Win XP: C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Logs + Vista: C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Logs + + On Windows the only suggestion in the MSDN docs is that local settings + go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in + examples of what some windows apps use for a logs dir.) + + OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA` + value for Windows and appends "log" to the user cache dir for Unix. + This can be disabled with the `opinion=False` option. + """ + if system == "darwin": + path = os.path.join( + os.path.expanduser('~/Library/Logs'), + appname) + elif system == "win32": + path = user_data_dir(appname, appauthor, version) + version = False + if opinion: + path = os.path.join(path, "Logs") + else: + path = user_cache_dir(appname, appauthor, version) + version = False + if opinion: + path = os.path.join(path, "log") + if appname and version: + path = os.path.join(path, version) + return path + + +class AppDirs(object): + """Convenience wrapper for getting application dirs.""" + def __init__(self, appname, appauthor=None, version=None, roaming=False, + multipath=False): + self.appname = appname + self.appauthor = appauthor + self.version = version + self.roaming = roaming + self.multipath = multipath + + @property + def user_data_dir(self): + return user_data_dir(self.appname, self.appauthor, + version=self.version, roaming=self.roaming) + + @property + def site_data_dir(self): + return site_data_dir(self.appname, self.appauthor, + version=self.version, multipath=self.multipath) + + @property + def user_config_dir(self): + return user_config_dir(self.appname, self.appauthor, + version=self.version, roaming=self.roaming) + + @property + def site_config_dir(self): + return site_config_dir(self.appname, self.appauthor, + version=self.version, multipath=self.multipath) + + @property + def user_cache_dir(self): + return user_cache_dir(self.appname, self.appauthor, + version=self.version) + + @property + def user_log_dir(self): + return user_log_dir(self.appname, self.appauthor, + version=self.version) + + +#---- internal support stuff + +def _get_win_folder_from_registry(csidl_name): + """This is a fallback technique at best. I'm not sure if using the + registry for this guarantees us the correct answer for all CSIDL_* + names. + """ + import _winreg + + shell_folder_name = { + "CSIDL_APPDATA": "AppData", + "CSIDL_COMMON_APPDATA": "Common AppData", + "CSIDL_LOCAL_APPDATA": "Local AppData", + }[csidl_name] + + key = _winreg.OpenKey( + _winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" + ) + dir, type = _winreg.QueryValueEx(key, shell_folder_name) + return dir + + +def _get_win_folder_with_pywin32(csidl_name): + from win32com.shell import shellcon, shell + dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0) + # Try to make this a unicode path because SHGetFolderPath does + # not return unicode strings when there is unicode data in the + # path. + try: + dir = unicode(dir) + + # Downgrade to short path name if have highbit chars. See + # <http://bugs.activestate.com/show_bug.cgi?id=85099>. + has_high_char = False + for c in dir: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + try: + import win32api + dir = win32api.GetShortPathName(dir) + except ImportError: + pass + except UnicodeError: + pass + return dir + + +def _get_win_folder_with_ctypes(csidl_name): + import ctypes + + csidl_const = { + "CSIDL_APPDATA": 26, + "CSIDL_COMMON_APPDATA": 35, + "CSIDL_LOCAL_APPDATA": 28, + }[csidl_name] + + buf = ctypes.create_unicode_buffer(1024) + ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) + + # Downgrade to short path name if have highbit chars. See + # <http://bugs.activestate.com/show_bug.cgi?id=85099>. + has_high_char = False + for c in buf: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + buf2 = ctypes.create_unicode_buffer(1024) + if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): + buf = buf2 + + return buf.value + +def _get_win_folder_with_jna(csidl_name): + import array + from com.sun import jna + from com.sun.jna.platform import win32 + + buf_size = win32.WinDef.MAX_PATH * 2 + buf = array.zeros('c', buf_size) + shell = win32.Shell32.INSTANCE + shell.SHGetFolderPath(None, getattr(win32.ShlObj, csidl_name), None, win32.ShlObj.SHGFP_TYPE_CURRENT, buf) + dir = jna.Native.toString(buf.tostring()).rstrip("\0") + + # Downgrade to short path name if have highbit chars. See + # <http://bugs.activestate.com/show_bug.cgi?id=85099>. + has_high_char = False + for c in dir: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + buf = array.zeros('c', buf_size) + kernel = win32.Kernel32.INSTANCE + if kernal.GetShortPathName(dir, buf, buf_size): + dir = jna.Native.toString(buf.tostring()).rstrip("\0") + + return dir + +if system == "win32": + try: + import win32com.shell + _get_win_folder = _get_win_folder_with_pywin32 + except ImportError: + try: + from ctypes import windll + _get_win_folder = _get_win_folder_with_ctypes + except ImportError: + try: + import com.sun.jna + _get_win_folder = _get_win_folder_with_jna + except ImportError: + _get_win_folder = _get_win_folder_from_registry + + +#---- self test code + +if __name__ == "__main__": + appname = "MyApp" + appauthor = "MyCompany" + + props = ("user_data_dir", "site_data_dir", + "user_config_dir", "site_config_dir", + "user_cache_dir", "user_log_dir") + + print("-- app dirs (with optional 'version')") + dirs = AppDirs(appname, appauthor, version="1.0") + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (without optional 'version')") + dirs = AppDirs(appname, appauthor) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (without optional 'appauthor')") + dirs = AppDirs(appname) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (with disabled 'appauthor')") + dirs = AppDirs(appname, appauthor=False) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) diff --git a/Contents/Libraries/Shared/argparse.py b/Contents/Libraries/Shared/argparse.py new file mode 100644 index 000000000..70a77cc02 --- /dev/null +++ b/Contents/Libraries/Shared/argparse.py @@ -0,0 +1,2392 @@ +# Author: Steven J. Bethard <steven.bethard@gmail.com>. +# Maintainer: Thomas Waldmann <tw@waldmann-edv.de> + +"""Command-line parsing library + +This module is an optparse-inspired command-line parsing library that: + + - handles both optional and positional arguments + - produces highly informative usage messages + - supports parsers that dispatch to sub-parsers + +The following is a simple usage example that sums integers from the +command-line and writes the result to a file:: + + parser = argparse.ArgumentParser( + description='sum the integers at the command line') + parser.add_argument( + 'integers', metavar='int', nargs='+', type=int, + help='an integer to be summed') + parser.add_argument( + '--log', default=sys.stdout, type=argparse.FileType('w'), + help='the file where the sum should be written') + args = parser.parse_args() + args.log.write('%s' % sum(args.integers)) + args.log.close() + +The module contains the following public classes: + + - ArgumentParser -- The main entry point for command-line parsing. As the + example above shows, the add_argument() method is used to populate + the parser with actions for optional and positional arguments. Then + the parse_args() method is invoked to convert the args at the + command-line into an object with attributes. + + - ArgumentError -- The exception raised by ArgumentParser objects when + there are errors with the parser's actions. Errors raised while + parsing the command-line are caught by ArgumentParser and emitted + as command-line messages. + + - FileType -- A factory for defining types of files to be created. As the + example above shows, instances of FileType are typically passed as + the type= argument of add_argument() calls. + + - Action -- The base class for parser actions. Typically actions are + selected by passing strings like 'store_true' or 'append_const' to + the action= argument of add_argument(). However, for greater + customization of ArgumentParser actions, subclasses of Action may + be defined and passed as the action= argument. + + - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, + ArgumentDefaultsHelpFormatter -- Formatter classes which + may be passed as the formatter_class= argument to the + ArgumentParser constructor. HelpFormatter is the default, + RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser + not to change the formatting for help text, and + ArgumentDefaultsHelpFormatter adds information about argument defaults + to the help. + +All other classes in this module are considered implementation details. +(Also note that HelpFormatter and RawDescriptionHelpFormatter are only +considered public as object names -- the API of the formatter objects is +still considered an implementation detail.) +""" + +__version__ = '1.4.0' # we use our own version number independant of the + # one in stdlib and we release this on pypi. + +__external_lib__ = True # to make sure the tests really test THIS lib, + # not the builtin one in Python stdlib + +__all__ = [ + 'ArgumentParser', + 'ArgumentError', + 'ArgumentTypeError', + 'FileType', + 'HelpFormatter', + 'ArgumentDefaultsHelpFormatter', + 'RawDescriptionHelpFormatter', + 'RawTextHelpFormatter', + 'Namespace', + 'Action', + 'ONE_OR_MORE', + 'OPTIONAL', + 'PARSER', + 'REMAINDER', + 'SUPPRESS', + 'ZERO_OR_MORE', +] + + +import copy as _copy +import os as _os +import re as _re +import sys as _sys +import textwrap as _textwrap + +from gettext import gettext as _ + +try: + set +except NameError: + # for python < 2.4 compatibility (sets module is there since 2.3): + from sets import Set as set + +try: + basestring +except NameError: + basestring = str + +try: + sorted +except NameError: + # for python < 2.4 compatibility: + def sorted(iterable, reverse=False): + result = list(iterable) + result.sort() + if reverse: + result.reverse() + return result + + +def _callable(obj): + return hasattr(obj, '__call__') or hasattr(obj, '__bases__') + + +SUPPRESS = '==SUPPRESS==' + +OPTIONAL = '?' +ZERO_OR_MORE = '*' +ONE_OR_MORE = '+' +PARSER = 'A...' +REMAINDER = '...' +_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args' + +# ============================= +# Utility functions and classes +# ============================= + +class _AttributeHolder(object): + """Abstract base class that provides __repr__. + + The __repr__ method returns a string in the format:: + ClassName(attr=name, attr=name, ...) + The attributes are determined either by a class-level attribute, + '_kwarg_names', or by inspecting the instance __dict__. + """ + + def __repr__(self): + type_name = type(self).__name__ + arg_strings = [] + for arg in self._get_args(): + arg_strings.append(repr(arg)) + for name, value in self._get_kwargs(): + arg_strings.append('%s=%r' % (name, value)) + return '%s(%s)' % (type_name, ', '.join(arg_strings)) + + def _get_kwargs(self): + return sorted(self.__dict__.items()) + + def _get_args(self): + return [] + + +def _ensure_value(namespace, name, value): + if getattr(namespace, name, None) is None: + setattr(namespace, name, value) + return getattr(namespace, name) + + +# =============== +# Formatting Help +# =============== + +class HelpFormatter(object): + """Formatter for generating usage messages and argument help strings. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def __init__(self, + prog, + indent_increment=2, + max_help_position=24, + width=None): + + # default setting for width + if width is None: + try: + width = int(_os.environ['COLUMNS']) + except (KeyError, ValueError): + width = 80 + width -= 2 + + self._prog = prog + self._indent_increment = indent_increment + self._max_help_position = max_help_position + self._width = width + + self._current_indent = 0 + self._level = 0 + self._action_max_length = 0 + + self._root_section = self._Section(self, None) + self._current_section = self._root_section + + self._whitespace_matcher = _re.compile(r'\s+') + self._long_break_matcher = _re.compile(r'\n\n\n+') + + # =============================== + # Section and indentation methods + # =============================== + def _indent(self): + self._current_indent += self._indent_increment + self._level += 1 + + def _dedent(self): + self._current_indent -= self._indent_increment + assert self._current_indent >= 0, 'Indent decreased below 0.' + self._level -= 1 + + class _Section(object): + + def __init__(self, formatter, parent, heading=None): + self.formatter = formatter + self.parent = parent + self.heading = heading + self.items = [] + + def format_help(self): + # format the indented section + if self.parent is not None: + self.formatter._indent() + join = self.formatter._join_parts + for func, args in self.items: + func(*args) + item_help = join([func(*args) for func, args in self.items]) + if self.parent is not None: + self.formatter._dedent() + + # return nothing if the section was empty + if not item_help: + return '' + + # add the heading if the section was non-empty + if self.heading is not SUPPRESS and self.heading is not None: + current_indent = self.formatter._current_indent + heading = '%*s%s:\n' % (current_indent, '', self.heading) + else: + heading = '' + + # join the section-initial newline, the heading and the help + return join(['\n', heading, item_help, '\n']) + + def _add_item(self, func, args): + self._current_section.items.append((func, args)) + + # ======================== + # Message building methods + # ======================== + def start_section(self, heading): + self._indent() + section = self._Section(self, self._current_section, heading) + self._add_item(section.format_help, []) + self._current_section = section + + def end_section(self): + self._current_section = self._current_section.parent + self._dedent() + + def add_text(self, text): + if text is not SUPPRESS and text is not None: + self._add_item(self._format_text, [text]) + + def add_usage(self, usage, actions, groups, prefix=None): + if usage is not SUPPRESS: + args = usage, actions, groups, prefix + self._add_item(self._format_usage, args) + + def add_argument(self, action): + if action.help is not SUPPRESS: + + # find all invocations + get_invocation = self._format_action_invocation + invocations = [get_invocation(action)] + for subaction in self._iter_indented_subactions(action): + invocations.append(get_invocation(subaction)) + + # update the maximum item length + invocation_length = max([len(s) for s in invocations]) + action_length = invocation_length + self._current_indent + self._action_max_length = max(self._action_max_length, + action_length) + + # add the item to the list + self._add_item(self._format_action, [action]) + + def add_arguments(self, actions): + for action in actions: + self.add_argument(action) + + # ======================= + # Help-formatting methods + # ======================= + def format_help(self): + help = self._root_section.format_help() + if help: + help = self._long_break_matcher.sub('\n\n', help) + help = help.strip('\n') + '\n' + return help + + def _join_parts(self, part_strings): + return ''.join([part + for part in part_strings + if part and part is not SUPPRESS]) + + def _format_usage(self, usage, actions, groups, prefix): + if prefix is None: + prefix = _('usage: ') + + # if usage is specified, use that + if usage is not None: + usage = usage % dict(prog=self._prog) + + # if no optionals or positionals are available, usage is just prog + elif usage is None and not actions: + usage = '%(prog)s' % dict(prog=self._prog) + + # if optionals and positionals are available, calculate usage + elif usage is None: + prog = '%(prog)s' % dict(prog=self._prog) + + # split optionals from positionals + optionals = [] + positionals = [] + for action in actions: + if action.option_strings: + optionals.append(action) + else: + positionals.append(action) + + # build full usage string + format = self._format_actions_usage + action_usage = format(optionals + positionals, groups) + usage = ' '.join([s for s in [prog, action_usage] if s]) + + # wrap the usage parts if it's too long + text_width = self._width - self._current_indent + if len(prefix) + len(usage) > text_width: + + # break usage into wrappable parts + part_regexp = r'\(.*?\)+|\[.*?\]+|\S+' + opt_usage = format(optionals, groups) + pos_usage = format(positionals, groups) + opt_parts = _re.findall(part_regexp, opt_usage) + pos_parts = _re.findall(part_regexp, pos_usage) + assert ' '.join(opt_parts) == opt_usage + assert ' '.join(pos_parts) == pos_usage + + # helper for wrapping lines + def get_lines(parts, indent, prefix=None): + lines = [] + line = [] + if prefix is not None: + line_len = len(prefix) - 1 + else: + line_len = len(indent) - 1 + for part in parts: + if line_len + 1 + len(part) > text_width: + lines.append(indent + ' '.join(line)) + line = [] + line_len = len(indent) - 1 + line.append(part) + line_len += len(part) + 1 + if line: + lines.append(indent + ' '.join(line)) + if prefix is not None: + lines[0] = lines[0][len(indent):] + return lines + + # if prog is short, follow it with optionals or positionals + if len(prefix) + len(prog) <= 0.75 * text_width: + indent = ' ' * (len(prefix) + len(prog) + 1) + if opt_parts: + lines = get_lines([prog] + opt_parts, indent, prefix) + lines.extend(get_lines(pos_parts, indent)) + elif pos_parts: + lines = get_lines([prog] + pos_parts, indent, prefix) + else: + lines = [prog] + + # if prog is long, put it on its own line + else: + indent = ' ' * len(prefix) + parts = opt_parts + pos_parts + lines = get_lines(parts, indent) + if len(lines) > 1: + lines = [] + lines.extend(get_lines(opt_parts, indent)) + lines.extend(get_lines(pos_parts, indent)) + lines = [prog] + lines + + # join lines into usage + usage = '\n'.join(lines) + + # prefix with 'usage:' + return '%s%s\n\n' % (prefix, usage) + + def _format_actions_usage(self, actions, groups): + # find group indices and identify actions in groups + group_actions = set() + inserts = {} + for group in groups: + try: + start = actions.index(group._group_actions[0]) + except ValueError: + continue + else: + end = start + len(group._group_actions) + if actions[start:end] == group._group_actions: + for action in group._group_actions: + group_actions.add(action) + if not group.required: + if start in inserts: + inserts[start] += ' [' + else: + inserts[start] = '[' + inserts[end] = ']' + else: + if start in inserts: + inserts[start] += ' (' + else: + inserts[start] = '(' + inserts[end] = ')' + for i in range(start + 1, end): + inserts[i] = '|' + + # collect all actions format strings + parts = [] + for i, action in enumerate(actions): + + # suppressed arguments are marked with None + # remove | separators for suppressed arguments + if action.help is SUPPRESS: + parts.append(None) + if inserts.get(i) == '|': + inserts.pop(i) + elif inserts.get(i + 1) == '|': + inserts.pop(i + 1) + + # produce all arg strings + elif not action.option_strings: + part = self._format_args(action, action.dest) + + # if it's in a group, strip the outer [] + if action in group_actions: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + + # add the action string to the list + parts.append(part) + + # produce the first way to invoke the option in brackets + else: + option_string = action.option_strings[0] + + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = '%s' % option_string + + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + part = '%s %s' % (option_string, args_string) + + # make it look optional if it's not required or in a group + if not action.required and action not in group_actions: + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + # insert things at the necessary indices + for i in sorted(inserts, reverse=True): + parts[i:i] = [inserts[i]] + + # join all the action items with spaces + text = ' '.join([item for item in parts if item is not None]) + + # clean up separators for mutually exclusive groups + open = r'[\[(]' + close = r'[\])]' + text = _re.sub(r'(%s) ' % open, r'\1', text) + text = _re.sub(r' (%s)' % close, r'\1', text) + text = _re.sub(r'%s *%s' % (open, close), r'', text) + text = _re.sub(r'\(([^|]*)\)', r'\1', text) + text = text.strip() + + # return the text + return text + + def _format_text(self, text): + if '%(prog)' in text: + text = text % dict(prog=self._prog) + text_width = self._width - self._current_indent + indent = ' ' * self._current_indent + return self._fill_text(text, text_width, indent) + '\n\n' + + def _format_action(self, action): + # determine the required width and the entry label + help_position = min(self._action_max_length + 2, + self._max_help_position) + help_width = self._width - help_position + action_width = help_position - self._current_indent - 2 + action_header = self._format_action_invocation(action) + + # ho nelp; start on same line and add a final newline + if not action.help: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + + # short action name; start on the same line and pad two spaces + elif len(action_header) <= action_width: + tup = self._current_indent, '', action_width, action_header + action_header = '%*s%-*s ' % tup + indent_first = 0 + + # long action name; start on the next line + else: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + indent_first = help_position + + # collect the pieces of the action help + parts = [action_header] + + # if there was help for the action, add lines of help text + if action.help: + help_text = self._expand_help(action) + help_lines = self._split_lines(help_text, help_width) + parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) + for line in help_lines[1:]: + parts.append('%*s%s\n' % (help_position, '', line)) + + # or add a newline if the description doesn't end with one + elif not action_header.endswith('\n'): + parts.append('\n') + + # if there are any sub-actions, add their help as well + for subaction in self._iter_indented_subactions(action): + parts.append(self._format_action(subaction)) + + # return a single string + return self._join_parts(parts) + + def _format_action_invocation(self, action): + if not action.option_strings: + metavar, = self._metavar_formatter(action, action.dest)(1) + return metavar + + else: + parts = [] + + # if the Optional doesn't take a value, format is: + # -s, --long + if action.nargs == 0: + parts.extend(action.option_strings) + + # if the Optional takes a value, format is: + # -s ARGS, --long ARGS + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + for option_string in action.option_strings: + parts.append('%s %s' % (option_string, args_string)) + + return ', '.join(parts) + + def _metavar_formatter(self, action, default_metavar): + if action.metavar is not None: + result = action.metavar + elif action.choices is not None: + choice_strs = [str(choice) for choice in action.choices] + result = '{%s}' % ','.join(choice_strs) + else: + result = default_metavar + + def format(tuple_size): + if isinstance(result, tuple): + return result + else: + return (result, ) * tuple_size + return format + + def _format_args(self, action, default_metavar): + get_metavar = self._metavar_formatter(action, default_metavar) + if action.nargs is None: + result = '%s' % get_metavar(1) + elif action.nargs == OPTIONAL: + result = '[%s]' % get_metavar(1) + elif action.nargs == ZERO_OR_MORE: + result = '[%s [%s ...]]' % get_metavar(2) + elif action.nargs == ONE_OR_MORE: + result = '%s [%s ...]' % get_metavar(2) + elif action.nargs == REMAINDER: + result = '...' + elif action.nargs == PARSER: + result = '%s ...' % get_metavar(1) + else: + formats = ['%s' for _ in range(action.nargs)] + result = ' '.join(formats) % get_metavar(action.nargs) + return result + + def _expand_help(self, action): + params = dict(vars(action), prog=self._prog) + for name in list(params): + if params[name] is SUPPRESS: + del params[name] + for name in list(params): + if hasattr(params[name], '__name__'): + params[name] = params[name].__name__ + if params.get('choices') is not None: + choices_str = ', '.join([str(c) for c in params['choices']]) + params['choices'] = choices_str + return self._get_help_string(action) % params + + def _iter_indented_subactions(self, action): + try: + get_subactions = action._get_subactions + except AttributeError: + pass + else: + self._indent() + for subaction in get_subactions(): + yield subaction + self._dedent() + + def _split_lines(self, text, width): + text = self._whitespace_matcher.sub(' ', text).strip() + return _textwrap.wrap(text, width) + + def _fill_text(self, text, width, indent): + text = self._whitespace_matcher.sub(' ', text).strip() + return _textwrap.fill(text, width, initial_indent=indent, + subsequent_indent=indent) + + def _get_help_string(self, action): + return action.help + + +class RawDescriptionHelpFormatter(HelpFormatter): + """Help message formatter which retains any formatting in descriptions. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _fill_text(self, text, width, indent): + return ''.join([indent + line for line in text.splitlines(True)]) + + +class RawTextHelpFormatter(RawDescriptionHelpFormatter): + """Help message formatter which retains formatting of all help text. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _split_lines(self, text, width): + return text.splitlines() + + +class ArgumentDefaultsHelpFormatter(HelpFormatter): + """Help message formatter which adds default values to argument help. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _get_help_string(self, action): + help = action.help + if '%(default)' not in action.help: + if action.default is not SUPPRESS: + defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + help += ' (default: %(default)s)' + return help + + +# ===================== +# Options and Arguments +# ===================== + +def _get_action_name(argument): + if argument is None: + return None + elif argument.option_strings: + return '/'.join(argument.option_strings) + elif argument.metavar not in (None, SUPPRESS): + return argument.metavar + elif argument.dest not in (None, SUPPRESS): + return argument.dest + else: + return None + + +class ArgumentError(Exception): + """An error from creating or using an argument (optional or positional). + + The string value of this exception is the message, augmented with + information about the argument that caused it. + """ + + def __init__(self, argument, message): + self.argument_name = _get_action_name(argument) + self.message = message + + def __str__(self): + if self.argument_name is None: + format = '%(message)s' + else: + format = 'argument %(argument_name)s: %(message)s' + return format % dict(message=self.message, + argument_name=self.argument_name) + + +class ArgumentTypeError(Exception): + """An error from trying to convert a command line string to a type.""" + pass + + +# ============== +# Action classes +# ============== + +class Action(_AttributeHolder): + """Information about how to convert command line strings to Python objects. + + Action objects are used by an ArgumentParser to represent the information + needed to parse a single argument from one or more strings from the + command line. The keyword arguments to the Action constructor are also + all attributes of Action instances. + + Keyword Arguments: + + - option_strings -- A list of command-line option strings which + should be associated with this action. + + - dest -- The name of the attribute to hold the created object(s) + + - nargs -- The number of command-line arguments that should be + consumed. By default, one argument will be consumed and a single + value will be produced. Other values include: + - N (an integer) consumes N arguments (and produces a list) + - '?' consumes zero or one arguments + - '*' consumes zero or more arguments (and produces a list) + - '+' consumes one or more arguments (and produces a list) + Note that the difference between the default and nargs=1 is that + with the default, a single value will be produced, while with + nargs=1, a list containing a single value will be produced. + + - const -- The value to be produced if the option is specified and the + option uses an action that takes no values. + + - default -- The value to be produced if the option is not specified. + + - type -- The type which the command-line arguments should be converted + to, should be one of 'string', 'int', 'float', 'complex' or a + callable object that accepts a single string argument. If None, + 'string' is assumed. + + - choices -- A container of values that should be allowed. If not None, + after a command-line argument has been converted to the appropriate + type, an exception will be raised if it is not a member of this + collection. + + - required -- True if the action must always be specified at the + command line. This is only meaningful for optional command-line + arguments. + + - help -- The help string describing the argument. + + - metavar -- The name to be used for the option's argument with the + help string. If None, the 'dest' value will be used as the name. + """ + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + self.option_strings = option_strings + self.dest = dest + self.nargs = nargs + self.const = const + self.default = default + self.type = type + self.choices = choices + self.required = required + self.help = help + self.metavar = metavar + + def _get_kwargs(self): + names = [ + 'option_strings', + 'dest', + 'nargs', + 'const', + 'default', + 'type', + 'choices', + 'help', + 'metavar', + ] + return [(name, getattr(self, name)) for name in names] + + def __call__(self, parser, namespace, values, option_string=None): + raise NotImplementedError(_('.__call__() not defined')) + + +class _StoreAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for store actions must be > 0; if you ' + 'have nothing to store, actions such as store ' + 'true or store const may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_StoreAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + + +class _StoreConstAction(Action): + + def __init__(self, + option_strings, + dest, + const, + default=None, + required=False, + help=None, + metavar=None): + super(_StoreConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, self.const) + + +class _StoreTrueAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=False, + required=False, + help=None): + super(_StoreTrueAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=True, + default=default, + required=required, + help=help) + + +class _StoreFalseAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=True, + required=False, + help=None): + super(_StoreFalseAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=False, + default=default, + required=required, + help=help) + + +class _AppendAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for append actions must be > 0; if arg ' + 'strings are not supplying the value to append, ' + 'the append const action may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_AppendAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = _copy.copy(_ensure_value(namespace, self.dest, [])) + items.append(values) + setattr(namespace, self.dest, items) + + +class _AppendConstAction(Action): + + def __init__(self, + option_strings, + dest, + const, + default=None, + required=False, + help=None, + metavar=None): + super(_AppendConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = _copy.copy(_ensure_value(namespace, self.dest, [])) + items.append(self.const) + setattr(namespace, self.dest, items) + + +class _CountAction(Action): + + def __init__(self, + option_strings, + dest, + default=None, + required=False, + help=None): + super(_CountAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + new_count = _ensure_value(namespace, self.dest, 0) + 1 + setattr(namespace, self.dest, new_count) + + +class _HelpAction(Action): + + def __init__(self, + option_strings, + dest=SUPPRESS, + default=SUPPRESS, + help=None): + super(_HelpAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + parser.print_help() + parser.exit() + + +class _VersionAction(Action): + + def __init__(self, + option_strings, + version=None, + dest=SUPPRESS, + default=SUPPRESS, + help="show program's version number and exit"): + super(_VersionAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + self.version = version + + def __call__(self, parser, namespace, values, option_string=None): + version = self.version + if version is None: + version = parser.version + formatter = parser._get_formatter() + formatter.add_text(version) + parser.exit(message=formatter.format_help()) + + +class _SubParsersAction(Action): + + class _ChoicesPseudoAction(Action): + + def __init__(self, name, aliases, help): + metavar = dest = name + if aliases: + metavar += ' (%s)' % ', '.join(aliases) + sup = super(_SubParsersAction._ChoicesPseudoAction, self) + sup.__init__(option_strings=[], dest=dest, help=help, + metavar=metavar) + + def __init__(self, + option_strings, + prog, + parser_class, + dest=SUPPRESS, + help=None, + metavar=None): + + self._prog_prefix = prog + self._parser_class = parser_class + self._name_parser_map = {} + self._choices_actions = [] + + super(_SubParsersAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=PARSER, + choices=self._name_parser_map, + help=help, + metavar=metavar) + + def add_parser(self, name, **kwargs): + # set prog from the existing prefix + if kwargs.get('prog') is None: + kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + + aliases = kwargs.pop('aliases', ()) + + # create a pseudo-action to hold the choice help + if 'help' in kwargs: + help = kwargs.pop('help') + choice_action = self._ChoicesPseudoAction(name, aliases, help) + self._choices_actions.append(choice_action) + + # create the parser and add it to the map + parser = self._parser_class(**kwargs) + self._name_parser_map[name] = parser + + # make parser available under aliases also + for alias in aliases: + self._name_parser_map[alias] = parser + + return parser + + def _get_subactions(self): + return self._choices_actions + + def __call__(self, parser, namespace, values, option_string=None): + parser_name = values[0] + arg_strings = values[1:] + + # set the parser name if requested + if self.dest is not SUPPRESS: + setattr(namespace, self.dest, parser_name) + + # select the parser + try: + parser = self._name_parser_map[parser_name] + except KeyError: + tup = parser_name, ', '.join(self._name_parser_map) + msg = _('unknown parser %r (choices: %s)' % tup) + raise ArgumentError(self, msg) + + # parse all the remaining options into the namespace + # store any unrecognized options on the object, so that the top + # level parser can decide what to do with them + namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + if arg_strings: + vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) + + +# ============== +# Type classes +# ============== + +class FileType(object): + """Factory for creating file object types + + Instances of FileType are typically passed as type= arguments to the + ArgumentParser add_argument() method. + + Keyword Arguments: + - mode -- A string indicating how the file is to be opened. Accepts the + same values as the builtin open() function. + - bufsize -- The file's desired buffer size. Accepts the same values as + the builtin open() function. + """ + + def __init__(self, mode='r', bufsize=None): + self._mode = mode + self._bufsize = bufsize + + def __call__(self, string): + # the special argument "-" means sys.std{in,out} + if string == '-': + if 'r' in self._mode: + return _sys.stdin + elif 'w' in self._mode: + return _sys.stdout + else: + msg = _('argument "-" with mode %r' % self._mode) + raise ValueError(msg) + + try: + # all other arguments are used as file names + if self._bufsize: + return open(string, self._mode, self._bufsize) + else: + return open(string, self._mode) + except IOError: + err = _sys.exc_info()[1] + message = _("can't open '%s': %s") + raise ArgumentTypeError(message % (string, err)) + + def __repr__(self): + args = [self._mode, self._bufsize] + args_str = ', '.join([repr(arg) for arg in args if arg is not None]) + return '%s(%s)' % (type(self).__name__, args_str) + +# =========================== +# Optional and Positional Parsing +# =========================== + +class Namespace(_AttributeHolder): + """Simple object for storing attributes. + + Implements equality by attribute names and values, and provides a simple + string representation. + """ + + def __init__(self, **kwargs): + for name in kwargs: + setattr(self, name, kwargs[name]) + + __hash__ = None + + def __eq__(self, other): + return vars(self) == vars(other) + + def __ne__(self, other): + return not (self == other) + + def __contains__(self, key): + return key in self.__dict__ + + +class _ActionsContainer(object): + + def __init__(self, + description, + prefix_chars, + argument_default, + conflict_handler): + super(_ActionsContainer, self).__init__() + + self.description = description + self.argument_default = argument_default + self.prefix_chars = prefix_chars + self.conflict_handler = conflict_handler + + # set up registries + self._registries = {} + + # register actions + self.register('action', None, _StoreAction) + self.register('action', 'store', _StoreAction) + self.register('action', 'store_const', _StoreConstAction) + self.register('action', 'store_true', _StoreTrueAction) + self.register('action', 'store_false', _StoreFalseAction) + self.register('action', 'append', _AppendAction) + self.register('action', 'append_const', _AppendConstAction) + self.register('action', 'count', _CountAction) + self.register('action', 'help', _HelpAction) + self.register('action', 'version', _VersionAction) + self.register('action', 'parsers', _SubParsersAction) + + # raise an exception if the conflict handler is invalid + self._get_handler() + + # action storage + self._actions = [] + self._option_string_actions = {} + + # groups + self._action_groups = [] + self._mutually_exclusive_groups = [] + + # defaults storage + self._defaults = {} + + # determines whether an "option" looks like a negative number + self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + + # whether or not there are any optionals that look like negative + # numbers -- uses a list so it can be shared and edited + self._has_negative_number_optionals = [] + + # ==================== + # Registration methods + # ==================== + def register(self, registry_name, value, object): + registry = self._registries.setdefault(registry_name, {}) + registry[value] = object + + def _registry_get(self, registry_name, value, default=None): + return self._registries[registry_name].get(value, default) + + # ================================== + # Namespace default accessor methods + # ================================== + def set_defaults(self, **kwargs): + self._defaults.update(kwargs) + + # if these defaults match any existing arguments, replace + # the previous default on the object with the new one + for action in self._actions: + if action.dest in kwargs: + action.default = kwargs[action.dest] + + def get_default(self, dest): + for action in self._actions: + if action.dest == dest and action.default is not None: + return action.default + return self._defaults.get(dest, None) + + + # ======================= + # Adding argument actions + # ======================= + def add_argument(self, *args, **kwargs): + """ + add_argument(dest, ..., name=value, ...) + add_argument(option_string, option_string, ..., name=value, ...) + """ + + # if no positional args are supplied or only one is supplied and + # it doesn't look like an option string, parse a positional + # argument + chars = self.prefix_chars + if not args or len(args) == 1 and args[0][0] not in chars: + if args and 'dest' in kwargs: + raise ValueError('dest supplied twice for positional argument') + kwargs = self._get_positional_kwargs(*args, **kwargs) + + # otherwise, we're adding an optional argument + else: + kwargs = self._get_optional_kwargs(*args, **kwargs) + + # if no default was supplied, use the parser-level default + if 'default' not in kwargs: + dest = kwargs['dest'] + if dest in self._defaults: + kwargs['default'] = self._defaults[dest] + elif self.argument_default is not None: + kwargs['default'] = self.argument_default + + # create the action object, and add it to the parser + action_class = self._pop_action_class(kwargs) + if not _callable(action_class): + raise ValueError('unknown action "%s"' % action_class) + action = action_class(**kwargs) + + # raise an error if the action type is not callable + type_func = self._registry_get('type', action.type, action.type) + if not _callable(type_func): + raise ValueError('%r is not callable' % type_func) + + return self._add_action(action) + + def add_argument_group(self, *args, **kwargs): + group = _ArgumentGroup(self, *args, **kwargs) + self._action_groups.append(group) + return group + + def add_mutually_exclusive_group(self, **kwargs): + group = _MutuallyExclusiveGroup(self, **kwargs) + self._mutually_exclusive_groups.append(group) + return group + + def _add_action(self, action): + # resolve any conflicts + self._check_conflict(action) + + # add to actions list + self._actions.append(action) + action.container = self + + # index the action by any option strings it has + for option_string in action.option_strings: + self._option_string_actions[option_string] = action + + # set the flag if any option strings look like negative numbers + for option_string in action.option_strings: + if self._negative_number_matcher.match(option_string): + if not self._has_negative_number_optionals: + self._has_negative_number_optionals.append(True) + + # return the created action + return action + + def _remove_action(self, action): + self._actions.remove(action) + + def _add_container_actions(self, container): + # collect groups by titles + title_group_map = {} + for group in self._action_groups: + if group.title in title_group_map: + msg = _('cannot merge actions - two groups are named %r') + raise ValueError(msg % (group.title)) + title_group_map[group.title] = group + + # map each action to its group + group_map = {} + for group in container._action_groups: + + # if a group with the title exists, use that, otherwise + # create a new group matching the container's group + if group.title not in title_group_map: + title_group_map[group.title] = self.add_argument_group( + title=group.title, + description=group.description, + conflict_handler=group.conflict_handler) + + # map the actions to their new group + for action in group._group_actions: + group_map[action] = title_group_map[group.title] + + # add container's mutually exclusive groups + # NOTE: if add_mutually_exclusive_group ever gains title= and + # description= then this code will need to be expanded as above + for group in container._mutually_exclusive_groups: + mutex_group = self.add_mutually_exclusive_group( + required=group.required) + + # map the actions to their new mutex group + for action in group._group_actions: + group_map[action] = mutex_group + + # add all actions to this container or their group + for action in container._actions: + group_map.get(action, self)._add_action(action) + + def _get_positional_kwargs(self, dest, **kwargs): + # make sure required is not specified + if 'required' in kwargs: + msg = _("'required' is an invalid argument for positionals") + raise TypeError(msg) + + # mark positional arguments as required if at least one is + # always required + if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: + kwargs['required'] = True + if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: + kwargs['required'] = True + + # return the keyword arguments with no option strings + return dict(kwargs, dest=dest, option_strings=[]) + + def _get_optional_kwargs(self, *args, **kwargs): + # determine short and long option strings + option_strings = [] + long_option_strings = [] + for option_string in args: + # error on strings that don't start with an appropriate prefix + if not option_string[0] in self.prefix_chars: + msg = _('invalid option string %r: ' + 'must start with a character %r') + tup = option_string, self.prefix_chars + raise ValueError(msg % tup) + + # strings starting with two prefix characters are long options + option_strings.append(option_string) + if option_string[0] in self.prefix_chars: + if len(option_string) > 1: + if option_string[1] in self.prefix_chars: + long_option_strings.append(option_string) + + # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' + dest = kwargs.pop('dest', None) + if dest is None: + if long_option_strings: + dest_option_string = long_option_strings[0] + else: + dest_option_string = option_strings[0] + dest = dest_option_string.lstrip(self.prefix_chars) + if not dest: + msg = _('dest= is required for options like %r') + raise ValueError(msg % option_string) + dest = dest.replace('-', '_') + + # return the updated keyword arguments + return dict(kwargs, dest=dest, option_strings=option_strings) + + def _pop_action_class(self, kwargs, default=None): + action = kwargs.pop('action', default) + return self._registry_get('action', action, action) + + def _get_handler(self): + # determine function from conflict handler string + handler_func_name = '_handle_conflict_%s' % self.conflict_handler + try: + return getattr(self, handler_func_name) + except AttributeError: + msg = _('invalid conflict_resolution value: %r') + raise ValueError(msg % self.conflict_handler) + + def _check_conflict(self, action): + + # find all options that conflict with this option + confl_optionals = [] + for option_string in action.option_strings: + if option_string in self._option_string_actions: + confl_optional = self._option_string_actions[option_string] + confl_optionals.append((option_string, confl_optional)) + + # resolve any conflicts + if confl_optionals: + conflict_handler = self._get_handler() + conflict_handler(action, confl_optionals) + + def _handle_conflict_error(self, action, conflicting_actions): + message = _('conflicting option string(s): %s') + conflict_string = ', '.join([option_string + for option_string, action + in conflicting_actions]) + raise ArgumentError(action, message % conflict_string) + + def _handle_conflict_resolve(self, action, conflicting_actions): + + # remove all conflicting options + for option_string, action in conflicting_actions: + + # remove the conflicting option + action.option_strings.remove(option_string) + self._option_string_actions.pop(option_string, None) + + # if the option now has no option string, remove it from the + # container holding it + if not action.option_strings: + action.container._remove_action(action) + + +class _ArgumentGroup(_ActionsContainer): + + def __init__(self, container, title=None, description=None, **kwargs): + # add any missing keyword arguments by checking the container + update = kwargs.setdefault + update('conflict_handler', container.conflict_handler) + update('prefix_chars', container.prefix_chars) + update('argument_default', container.argument_default) + super_init = super(_ArgumentGroup, self).__init__ + super_init(description=description, **kwargs) + + # group attributes + self.title = title + self._group_actions = [] + + # share most attributes with the container + self._registries = container._registries + self._actions = container._actions + self._option_string_actions = container._option_string_actions + self._defaults = container._defaults + self._has_negative_number_optionals = \ + container._has_negative_number_optionals + + def _add_action(self, action): + action = super(_ArgumentGroup, self)._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + super(_ArgumentGroup, self)._remove_action(action) + self._group_actions.remove(action) + + +class _MutuallyExclusiveGroup(_ArgumentGroup): + + def __init__(self, container, required=False): + super(_MutuallyExclusiveGroup, self).__init__(container) + self.required = required + self._container = container + + def _add_action(self, action): + if action.required: + msg = _('mutually exclusive arguments must be optional') + raise ValueError(msg) + action = self._container._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + self._container._remove_action(action) + self._group_actions.remove(action) + + +class ArgumentParser(_AttributeHolder, _ActionsContainer): + """Object for parsing command line strings into Python objects. + + Keyword Arguments: + - prog -- The name of the program (default: sys.argv[0]) + - usage -- A usage message (default: auto-generated from arguments) + - description -- A description of what the program does + - epilog -- Text following the argument descriptions + - parents -- Parsers whose arguments should be copied into this one + - formatter_class -- HelpFormatter class for printing help messages + - prefix_chars -- Characters that prefix optional arguments + - fromfile_prefix_chars -- Characters that prefix files containing + additional arguments + - argument_default -- The default value for all arguments + - conflict_handler -- String indicating how to handle conflicts + - add_help -- Add a -h/-help option + """ + + def __init__(self, + prog=None, + usage=None, + description=None, + epilog=None, + version=None, + parents=[], + formatter_class=HelpFormatter, + prefix_chars='-', + fromfile_prefix_chars=None, + argument_default=None, + conflict_handler='error', + add_help=True): + + if version is not None: + import warnings + warnings.warn( + """The "version" argument to ArgumentParser is deprecated. """ + """Please use """ + """"add_argument(..., action='version', version="N", ...)" """ + """instead""", DeprecationWarning) + + superinit = super(ArgumentParser, self).__init__ + superinit(description=description, + prefix_chars=prefix_chars, + argument_default=argument_default, + conflict_handler=conflict_handler) + + # default setting for prog + if prog is None: + prog = _os.path.basename(_sys.argv[0]) + + self.prog = prog + self.usage = usage + self.epilog = epilog + self.version = version + self.formatter_class = formatter_class + self.fromfile_prefix_chars = fromfile_prefix_chars + self.add_help = add_help + + add_group = self.add_argument_group + self._positionals = add_group(_('positional arguments')) + self._optionals = add_group(_('optional arguments')) + self._subparsers = None + + # register types + def identity(string): + return string + self.register('type', None, identity) + + # add help and version arguments if necessary + # (using explicit default to override global argument_default) + if '-' in prefix_chars: + default_prefix = '-' + else: + default_prefix = prefix_chars[0] + if self.add_help: + self.add_argument( + default_prefix+'h', default_prefix*2+'help', + action='help', default=SUPPRESS, + help=_('show this help message and exit')) + if self.version: + self.add_argument( + default_prefix+'v', default_prefix*2+'version', + action='version', default=SUPPRESS, + version=self.version, + help=_("show program's version number and exit")) + + # add parent arguments and defaults + for parent in parents: + self._add_container_actions(parent) + try: + defaults = parent._defaults + except AttributeError: + pass + else: + self._defaults.update(defaults) + + # ======================= + # Pretty __repr__ methods + # ======================= + def _get_kwargs(self): + names = [ + 'prog', + 'usage', + 'description', + 'version', + 'formatter_class', + 'conflict_handler', + 'add_help', + ] + return [(name, getattr(self, name)) for name in names] + + # ================================== + # Optional/Positional adding methods + # ================================== + def add_subparsers(self, **kwargs): + if self._subparsers is not None: + self.error(_('cannot have multiple subparser arguments')) + + # add the parser class to the arguments if it's not present + kwargs.setdefault('parser_class', type(self)) + + if 'title' in kwargs or 'description' in kwargs: + title = _(kwargs.pop('title', 'subcommands')) + description = _(kwargs.pop('description', None)) + self._subparsers = self.add_argument_group(title, description) + else: + self._subparsers = self._positionals + + # prog defaults to the usage message of this parser, skipping + # optional arguments and with no "usage:" prefix + if kwargs.get('prog') is None: + formatter = self._get_formatter() + positionals = self._get_positional_actions() + groups = self._mutually_exclusive_groups + formatter.add_usage(self.usage, positionals, groups, '') + kwargs['prog'] = formatter.format_help().strip() + + # create the parsers action and add it to the positionals list + parsers_class = self._pop_action_class(kwargs, 'parsers') + action = parsers_class(option_strings=[], **kwargs) + self._subparsers._add_action(action) + + # return the created parsers action + return action + + def _add_action(self, action): + if action.option_strings: + self._optionals._add_action(action) + else: + self._positionals._add_action(action) + return action + + def _get_optional_actions(self): + return [action + for action in self._actions + if action.option_strings] + + def _get_positional_actions(self): + return [action + for action in self._actions + if not action.option_strings] + + # ===================================== + # Command line argument parsing methods + # ===================================== + def parse_args(self, args=None, namespace=None): + args, argv = self.parse_known_args(args, namespace) + if argv: + msg = _('unrecognized arguments: %s') + self.error(msg % ' '.join(argv)) + return args + + def parse_known_args(self, args=None, namespace=None): + # args default to the system args + if args is None: + args = _sys.argv[1:] + + # default Namespace built from parser defaults + if namespace is None: + namespace = Namespace() + + # add any action defaults that aren't present + for action in self._actions: + if action.dest is not SUPPRESS: + if not hasattr(namespace, action.dest): + if action.default is not SUPPRESS: + setattr(namespace, action.dest, action.default) + + # add any parser defaults that aren't present + for dest in self._defaults: + if not hasattr(namespace, dest): + setattr(namespace, dest, self._defaults[dest]) + + # parse the arguments and exit if there are any errors + try: + namespace, args = self._parse_known_args(args, namespace) + if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) + delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) + return namespace, args + except ArgumentError: + err = _sys.exc_info()[1] + self.error(str(err)) + + def _parse_known_args(self, arg_strings, namespace): + # replace arg strings that are file references + if self.fromfile_prefix_chars is not None: + arg_strings = self._read_args_from_files(arg_strings) + + # map all mutually exclusive arguments to the other arguments + # they can't occur with + action_conflicts = {} + for mutex_group in self._mutually_exclusive_groups: + group_actions = mutex_group._group_actions + for i, mutex_action in enumerate(mutex_group._group_actions): + conflicts = action_conflicts.setdefault(mutex_action, []) + conflicts.extend(group_actions[:i]) + conflicts.extend(group_actions[i + 1:]) + + # find all option indices, and determine the arg_string_pattern + # which has an 'O' if there is an option at an index, + # an 'A' if there is an argument, or a '-' if there is a '--' + option_string_indices = {} + arg_string_pattern_parts = [] + arg_strings_iter = iter(arg_strings) + for i, arg_string in enumerate(arg_strings_iter): + + # all args after -- are non-options + if arg_string == '--': + arg_string_pattern_parts.append('-') + for arg_string in arg_strings_iter: + arg_string_pattern_parts.append('A') + + # otherwise, add the arg to the arg strings + # and note the index if it was an option + else: + option_tuple = self._parse_optional(arg_string) + if option_tuple is None: + pattern = 'A' + else: + option_string_indices[i] = option_tuple + pattern = 'O' + arg_string_pattern_parts.append(pattern) + + # join the pieces together to form the pattern + arg_strings_pattern = ''.join(arg_string_pattern_parts) + + # converts arg strings to the appropriate and then takes the action + seen_actions = set() + seen_non_default_actions = set() + + def take_action(action, argument_strings, option_string=None): + seen_actions.add(action) + argument_values = self._get_values(action, argument_strings) + + # error if this argument is not allowed with other previously + # seen arguments, assuming that actions that use the default + # value don't really count as "present" + if argument_values is not action.default: + seen_non_default_actions.add(action) + for conflict_action in action_conflicts.get(action, []): + if conflict_action in seen_non_default_actions: + msg = _('not allowed with argument %s') + action_name = _get_action_name(conflict_action) + raise ArgumentError(action, msg % action_name) + + # take the action if we didn't receive a SUPPRESS value + # (e.g. from a default) + if argument_values is not SUPPRESS: + action(self, namespace, argument_values, option_string) + + # function to convert arg_strings into an optional action + def consume_optional(start_index): + + # get the optional identified at this index + option_tuple = option_string_indices[start_index] + action, option_string, explicit_arg = option_tuple + + # identify additional optionals in the same arg string + # (e.g. -xyz is the same as -x -y -z if no args are required) + match_argument = self._match_argument + action_tuples = [] + while True: + + # if we found no optional action, skip it + if action is None: + extras.append(arg_strings[start_index]) + return start_index + 1 + + # if there is an explicit argument, try to match the + # optional's string arguments to only this + if explicit_arg is not None: + arg_count = match_argument(action, 'A') + + # if the action is a single-dash option and takes no + # arguments, try to parse more single-dash options out + # of the tail of the option string + chars = self.prefix_chars + if arg_count == 0 and option_string[1] not in chars: + action_tuples.append((action, [], option_string)) + char = option_string[0] + option_string = char + explicit_arg[0] + new_explicit_arg = explicit_arg[1:] or None + optionals_map = self._option_string_actions + if option_string in optionals_map: + action = optionals_map[option_string] + explicit_arg = new_explicit_arg + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if the action expect exactly one argument, we've + # successfully matched the option; exit the loop + elif arg_count == 1: + stop = start_index + 1 + args = [explicit_arg] + action_tuples.append((action, args, option_string)) + break + + # error if a double-dash option did not use the + # explicit argument + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if there is no explicit argument, try to match the + # optional's string arguments with the following strings + # if successful, exit the loop + else: + start = start_index + 1 + selected_patterns = arg_strings_pattern[start:] + arg_count = match_argument(action, selected_patterns) + stop = start + arg_count + args = arg_strings[start:stop] + action_tuples.append((action, args, option_string)) + break + + # add the Optional to the list and return the index at which + # the Optional's string args stopped + assert action_tuples + for action, args, option_string in action_tuples: + take_action(action, args, option_string) + return stop + + # the list of Positionals left to be parsed; this is modified + # by consume_positionals() + positionals = self._get_positional_actions() + + # function to convert arg_strings into positional actions + def consume_positionals(start_index): + # match as many Positionals as possible + match_partial = self._match_arguments_partial + selected_pattern = arg_strings_pattern[start_index:] + arg_counts = match_partial(positionals, selected_pattern) + + # slice off the appropriate arg strings for each Positional + # and add the Positional and its args to the list + for action, arg_count in zip(positionals, arg_counts): + args = arg_strings[start_index: start_index + arg_count] + start_index += arg_count + take_action(action, args) + + # slice off the Positionals that we just parsed and return the + # index at which the Positionals' string args stopped + positionals[:] = positionals[len(arg_counts):] + return start_index + + # consume Positionals and Optionals alternately, until we have + # passed the last option string + extras = [] + start_index = 0 + if option_string_indices: + max_option_string_index = max(option_string_indices) + else: + max_option_string_index = -1 + while start_index <= max_option_string_index: + + # consume any Positionals preceding the next option + next_option_string_index = min([ + index + for index in option_string_indices + if index >= start_index]) + if start_index != next_option_string_index: + positionals_end_index = consume_positionals(start_index) + + # only try to parse the next optional if we didn't consume + # the option string during the positionals parsing + if positionals_end_index > start_index: + start_index = positionals_end_index + continue + else: + start_index = positionals_end_index + + # if we consumed all the positionals we could and we're not + # at the index of an option string, there were extra arguments + if start_index not in option_string_indices: + strings = arg_strings[start_index:next_option_string_index] + extras.extend(strings) + start_index = next_option_string_index + + # consume the next optional and any arguments for it + start_index = consume_optional(start_index) + + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index) + + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + + # if we didn't use all the Positional objects, there were too few + # arg strings supplied. + if positionals: + self.error(_('too few arguments')) + + # make sure all required actions were present, and convert defaults. + for action in self._actions: + if action not in seen_actions: + if action.required: + name = _get_action_name(action) + self.error(_('argument %s is required') % name) + else: + # Convert action default now instead of doing it before + # parsing arguments to avoid calling convert functions + # twice (which may fail) if the argument was given, but + # only if it was defined already in the namespace + if (action.default is not None and + isinstance(action.default, basestring) and + hasattr(namespace, action.dest) and + action.default is getattr(namespace, action.dest)): + setattr(namespace, action.dest, + self._get_value(action, action.default)) + + # make sure all required groups had one option present + for group in self._mutually_exclusive_groups: + if group.required: + for action in group._group_actions: + if action in seen_non_default_actions: + break + + # if no actions were used, report the error + else: + names = [_get_action_name(action) + for action in group._group_actions + if action.help is not SUPPRESS] + msg = _('one of the arguments %s is required') + self.error(msg % ' '.join(names)) + + # return the updated namespace and the extra arguments + return namespace, extras + + def _read_args_from_files(self, arg_strings): + # expand arguments referencing files + new_arg_strings = [] + for arg_string in arg_strings: + + # for regular arguments, just add them back into the list + if arg_string[0] not in self.fromfile_prefix_chars: + new_arg_strings.append(arg_string) + + # replace arguments referencing files with the file content + else: + try: + args_file = open(arg_string[1:]) + try: + arg_strings = [] + for arg_line in args_file.read().splitlines(): + for arg in self.convert_arg_line_to_args(arg_line): + arg_strings.append(arg) + arg_strings = self._read_args_from_files(arg_strings) + new_arg_strings.extend(arg_strings) + finally: + args_file.close() + except IOError: + err = _sys.exc_info()[1] + self.error(str(err)) + + # return the modified argument list + return new_arg_strings + + def convert_arg_line_to_args(self, arg_line): + return [arg_line] + + def _match_argument(self, action, arg_strings_pattern): + # match the pattern for this action to the arg strings + nargs_pattern = self._get_nargs_pattern(action) + match = _re.match(nargs_pattern, arg_strings_pattern) + + # raise an exception if we weren't able to find a match + if match is None: + nargs_errors = { + None: _('expected one argument'), + OPTIONAL: _('expected at most one argument'), + ONE_OR_MORE: _('expected at least one argument'), + } + default = _('expected %s argument(s)') % action.nargs + msg = nargs_errors.get(action.nargs, default) + raise ArgumentError(action, msg) + + # return the number of arguments matched + return len(match.group(1)) + + def _match_arguments_partial(self, actions, arg_strings_pattern): + # progressively shorten the actions list by slicing off the + # final actions until we find a match + result = [] + for i in range(len(actions), 0, -1): + actions_slice = actions[:i] + pattern = ''.join([self._get_nargs_pattern(action) + for action in actions_slice]) + match = _re.match(pattern, arg_strings_pattern) + if match is not None: + result.extend([len(string) for string in match.groups()]) + break + + # return the list of arg string counts + return result + + def _parse_optional(self, arg_string): + # if it's an empty string, it was meant to be a positional + if not arg_string: + return None + + # if it doesn't start with a prefix, it was meant to be positional + if not arg_string[0] in self.prefix_chars: + return None + + # if the option string is present in the parser, return the action + if arg_string in self._option_string_actions: + action = self._option_string_actions[arg_string] + return action, arg_string, None + + # if it's just a single character, it was meant to be positional + if len(arg_string) == 1: + return None + + # if the option string before the "=" is present, return the action + if '=' in arg_string: + option_string, explicit_arg = arg_string.split('=', 1) + if option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, explicit_arg + + # search through all possible prefixes of the option string + # and all actions in the parser for possible interpretations + option_tuples = self._get_option_tuples(arg_string) + + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, explicit_arg in option_tuples]) + tup = arg_string, options + self.error(_('ambiguous option: %s could match %s') % tup) + + # if exactly one action matched, this segmentation is good, + # so return the parsed action + elif len(option_tuples) == 1: + option_tuple, = option_tuples + return option_tuple + + # if it was not found as an option, but it looks like a negative + # number, it was meant to be positional + # unless there are negative-number-like options + if self._negative_number_matcher.match(arg_string): + if not self._has_negative_number_optionals: + return None + + # if it contains a space, it was meant to be a positional + if ' ' in arg_string: + return None + + # it was meant to be an optional but there is no such option + # in this parser (though it might be a valid option in a subparser) + return None, arg_string, None + + def _get_option_tuples(self, option_string): + result = [] + + # option strings starting with two prefix characters are only + # split at the '=' + chars = self.prefix_chars + if option_string[0] in chars and option_string[1] in chars: + if '=' in option_string: + option_prefix, explicit_arg = option_string.split('=', 1) + else: + option_prefix = option_string + explicit_arg = None + for option_string in self._option_string_actions: + if option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, explicit_arg + result.append(tup) + + # single character options can be concatenated with their arguments + # but multiple character options always have to have their argument + # separate + elif option_string[0] in chars and option_string[1] not in chars: + option_prefix = option_string + explicit_arg = None + short_option_prefix = option_string[:2] + short_explicit_arg = option_string[2:] + + for option_string in self._option_string_actions: + if option_string == short_option_prefix: + action = self._option_string_actions[option_string] + tup = action, option_string, short_explicit_arg + result.append(tup) + elif option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, explicit_arg + result.append(tup) + + # shouldn't ever get here + else: + self.error(_('unexpected option string: %s') % option_string) + + # return the collected option tuples + return result + + def _get_nargs_pattern(self, action): + # in all examples below, we have to allow for '--' args + # which are represented as '-' in the pattern + nargs = action.nargs + + # the default (None) is assumed to be a single argument + if nargs is None: + nargs_pattern = '(-*A-*)' + + # allow zero or one arguments + elif nargs == OPTIONAL: + nargs_pattern = '(-*A?-*)' + + # allow zero or more arguments + elif nargs == ZERO_OR_MORE: + nargs_pattern = '(-*[A-]*)' + + # allow one or more arguments + elif nargs == ONE_OR_MORE: + nargs_pattern = '(-*A[A-]*)' + + # allow any number of options or arguments + elif nargs == REMAINDER: + nargs_pattern = '([-AO]*)' + + # allow one argument followed by any number of options or arguments + elif nargs == PARSER: + nargs_pattern = '(-*A[-AO]*)' + + # all others should be integers + else: + nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) + + # if this is an optional action, -- is not allowed + if action.option_strings: + nargs_pattern = nargs_pattern.replace('-*', '') + nargs_pattern = nargs_pattern.replace('-', '') + + # return the pattern + return nargs_pattern + + # ======================== + # Value conversion methods + # ======================== + def _get_values(self, action, arg_strings): + # for everything but PARSER args, strip out '--' + if action.nargs not in [PARSER, REMAINDER]: + arg_strings = [s for s in arg_strings if s != '--'] + + # optional argument produces a default when not present + if not arg_strings and action.nargs == OPTIONAL: + if action.option_strings: + value = action.const + else: + value = action.default + if isinstance(value, basestring): + value = self._get_value(action, value) + self._check_value(action, value) + + # when nargs='*' on a positional, if there were no command-line + # args, use the default if it is anything other than None + elif (not arg_strings and action.nargs == ZERO_OR_MORE and + not action.option_strings): + if action.default is not None: + value = action.default + else: + value = arg_strings + self._check_value(action, value) + + # single argument or optional argument produces a single value + elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: + arg_string, = arg_strings + value = self._get_value(action, arg_string) + self._check_value(action, value) + + # REMAINDER arguments convert all values, checking none + elif action.nargs == REMAINDER: + value = [self._get_value(action, v) for v in arg_strings] + + # PARSER arguments convert all values, but check only the first + elif action.nargs == PARSER: + value = [self._get_value(action, v) for v in arg_strings] + self._check_value(action, value[0]) + + # all other types of nargs produce a list + else: + value = [self._get_value(action, v) for v in arg_strings] + for v in value: + self._check_value(action, v) + + # return the converted value + return value + + def _get_value(self, action, arg_string): + type_func = self._registry_get('type', action.type, action.type) + if not _callable(type_func): + msg = _('%r is not callable') + raise ArgumentError(action, msg % type_func) + + # convert the value to the appropriate type + try: + result = type_func(arg_string) + + # ArgumentTypeErrors indicate errors + except ArgumentTypeError: + name = getattr(action.type, '__name__', repr(action.type)) + msg = str(_sys.exc_info()[1]) + raise ArgumentError(action, msg) + + # TypeErrors or ValueErrors also indicate errors + except (TypeError, ValueError): + name = getattr(action.type, '__name__', repr(action.type)) + msg = _('invalid %s value: %r') + raise ArgumentError(action, msg % (name, arg_string)) + + # return the converted value + return result + + def _check_value(self, action, value): + # converted value must be one of the choices (if specified) + if action.choices is not None and value not in action.choices: + tup = value, ', '.join(map(repr, action.choices)) + msg = _('invalid choice: %r (choose from %s)') % tup + raise ArgumentError(action, msg) + + # ======================= + # Help-formatting methods + # ======================= + def format_usage(self): + formatter = self._get_formatter() + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + return formatter.format_help() + + def format_help(self): + formatter = self._get_formatter() + + # usage + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + + # description + formatter.add_text(self.description) + + # positionals, optionals and user-defined groups + for action_group in self._action_groups: + formatter.start_section(action_group.title) + formatter.add_text(action_group.description) + formatter.add_arguments(action_group._group_actions) + formatter.end_section() + + # epilog + formatter.add_text(self.epilog) + + # determine help from format above + return formatter.format_help() + + def format_version(self): + import warnings + warnings.warn( + 'The format_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + formatter = self._get_formatter() + formatter.add_text(self.version) + return formatter.format_help() + + def _get_formatter(self): + return self.formatter_class(prog=self.prog) + + # ===================== + # Help-printing methods + # ===================== + def print_usage(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_usage(), file) + + def print_help(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_help(), file) + + def print_version(self, file=None): + import warnings + warnings.warn( + 'The print_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + self._print_message(self.format_version(), file) + + def _print_message(self, message, file=None): + if message: + if file is None: + file = _sys.stderr + file.write(message) + + # =============== + # Exiting methods + # =============== + def exit(self, status=0, message=None): + if message: + self._print_message(message, _sys.stderr) + _sys.exit(status) + + def error(self, message): + """error(message: string) + + Prints a usage message incorporating the message to stderr and + exits. + + If you override this in a subclass, it should not return -- it + should either exit or raise an exception. + """ + self.print_usage(_sys.stderr) + self.exit(2, _('%s: error: %s\n') % (self.prog, message)) diff --git a/Contents/Libraries/Shared/asio/__init__.py b/Contents/Libraries/Shared/asio/__init__.py new file mode 100644 index 000000000..ca8ded235 --- /dev/null +++ b/Contents/Libraries/Shared/asio/__init__.py @@ -0,0 +1,61 @@ +# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from asio.file import SEEK_ORIGIN_CURRENT +from asio.file_opener import FileOpener +from asio.open_parameters import OpenParameters +from asio.interfaces.posix import PosixInterface +from asio.interfaces.windows import WindowsInterface + +import os + + +class ASIO(object): + platform_handler = None + + @classmethod + def get_handler(cls): + if cls.platform_handler: + return cls.platform_handler + + if os.name == 'nt': + cls.platform_handler = WindowsInterface + elif os.name == 'posix': + cls.platform_handler = PosixInterface + else: + raise NotImplementedError() + + return cls.platform_handler + + @classmethod + def open(cls, file_path, opener=True, parameters=None): + """Open file + + :type file_path: str + + :param opener: Use FileOpener, for use with the 'with' statement + :type opener: bool + + :rtype: asio.file.File + """ + if not parameters: + parameters = OpenParameters() + + if opener: + return FileOpener(file_path, parameters) + + return ASIO.get_handler().open( + file_path, + parameters=parameters.handlers.get(ASIO.get_handler()) + ) diff --git a/Contents/Libraries/Shared/asio/file.py b/Contents/Libraries/Shared/asio/file.py new file mode 100644 index 000000000..a44970815 --- /dev/null +++ b/Contents/Libraries/Shared/asio/file.py @@ -0,0 +1,92 @@ +# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from io import RawIOBase +import time + +DEFAULT_BUFFER_SIZE = 4096 + +SEEK_ORIGIN_BEGIN = 0 +SEEK_ORIGIN_CURRENT = 1 +SEEK_ORIGIN_END = 2 + + +class ReadTimeoutError(Exception): + pass + + +class File(RawIOBase): + platform_handler = None + + def __init__(self, *args, **kwargs): + super(File, self).__init__(*args, **kwargs) + + def get_handler(self): + """ + :rtype: asio.interfaces.base.Interface + """ + if not self.platform_handler: + raise ValueError() + + return self.platform_handler + + def get_size(self): + """Get the current file size + + :rtype: int + """ + return self.get_handler().get_size(self) + + def get_path(self): + """Get the path of this file + + :rtype: str + """ + return self.get_handler().get_path(self) + + def seek(self, offset, origin): + """Sets a reference point of a file to the given value. + + :param offset: The point relative to origin to move + :type offset: int + + :param origin: Reference point to seek (SEEK_ORIGIN_BEGIN, SEEK_ORIGIN_CURRENT, SEEK_ORIGIN_END) + :type origin: int + """ + return self.get_handler().seek(self, offset, origin) + + def read(self, n=-1): + """Read up to n bytes from the object and return them. + + :type n: int + :rtype: str + """ + return self.get_handler().read(self, n) + + def readinto(self, b): + """Read up to len(b) bytes into bytearray b and return the number of bytes read.""" + data = self.read(len(b)) + + if data is None: + return None + + b[:len(data)] = data + return len(data) + + def close(self): + """Close the file handle""" + return self.get_handler().close(self) + + def readable(self, *args, **kwargs): + return True diff --git a/Contents/Libraries/Shared/asio/file_opener.py b/Contents/Libraries/Shared/asio/file_opener.py new file mode 100644 index 000000000..990cc9804 --- /dev/null +++ b/Contents/Libraries/Shared/asio/file_opener.py @@ -0,0 +1,21 @@ +class FileOpener(object): + def __init__(self, file_path, parameters=None): + self.file_path = file_path + self.parameters = parameters + + self.file = None + + def __enter__(self): + self.file = ASIO.get_handler().open( + self.file_path, + self.parameters.handlers.get(ASIO.get_handler()) + ) + + return self.file + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.file: + return + + self.file.close() + self.file = None diff --git a/Contents/Libraries/Shared/asio/interfaces/__init__.py b/Contents/Libraries/Shared/asio/interfaces/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Contents/Libraries/Shared/asio/interfaces/base.py b/Contents/Libraries/Shared/asio/interfaces/base.py new file mode 100644 index 000000000..6188b000f --- /dev/null +++ b/Contents/Libraries/Shared/asio/interfaces/base.py @@ -0,0 +1,41 @@ +# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from asio.file import DEFAULT_BUFFER_SIZE + + +class Interface(object): + @classmethod + def open(cls, file_path, parameters=None): + raise NotImplementedError() + + @classmethod + def get_size(cls, fp): + raise NotImplementedError() + + @classmethod + def get_path(cls, fp): + raise NotImplementedError() + + @classmethod + def seek(cls, fp, pointer, distance): + raise NotImplementedError() + + @classmethod + def read(cls, fp, n=DEFAULT_BUFFER_SIZE): + raise NotImplementedError() + + @classmethod + def close(cls, fp): + raise NotImplementedError() diff --git a/Contents/Libraries/Shared/asio/interfaces/posix.py b/Contents/Libraries/Shared/asio/interfaces/posix.py new file mode 100644 index 000000000..b235c02b9 --- /dev/null +++ b/Contents/Libraries/Shared/asio/interfaces/posix.py @@ -0,0 +1,123 @@ +# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from asio.file import File, DEFAULT_BUFFER_SIZE +from asio.interfaces.base import Interface + +import sys +import os + +if os.name == 'posix': + import select + + # fcntl is only required on darwin + if sys.platform == 'darwin': + import fcntl + +F_GETPATH = 50 + + +class PosixInterface(Interface): + @classmethod + def open(cls, file_path, parameters=None): + """ + :type file_path: str + :rtype: asio.interfaces.posix.PosixFile + """ + if not parameters: + parameters = {} + + if not parameters.get('mode'): + parameters.pop('mode') + + if not parameters.get('buffering'): + parameters.pop('buffering') + + fd = os.open(file_path, os.O_RDONLY | os.O_NONBLOCK) + + return PosixFile(fd) + + @classmethod + def get_size(cls, fp): + """ + :type fp: asio.interfaces.posix.PosixFile + :rtype: int + """ + return os.fstat(fp.fd).st_size + + @classmethod + def get_path(cls, fp): + """ + :type fp: asio.interfaces.posix.PosixFile + :rtype: int + """ + + # readlink /dev/fd fails on darwin, so instead use fcntl F_GETPATH + if sys.platform == 'darwin': + return fcntl.fcntl(fp.fd, F_GETPATH, '\0' * 1024).rstrip('\0') + + # Use /proc/self/fd if available + if os.path.lexists("/proc/self/fd/"): + return os.readlink("/proc/self/fd/%s" % fp.fd) + + # Fallback to /dev/fd + if os.path.lexists("/dev/fd/"): + return os.readlink("/dev/fd/%s" % fp.fd) + + raise NotImplementedError('Environment not supported (fdescfs not mounted?)') + + @classmethod + def seek(cls, fp, offset, origin): + """ + :type fp: asio.interfaces.posix.PosixFile + :type offset: int + :type origin: int + """ + os.lseek(fp.fd, offset, origin) + + @classmethod + def read(cls, fp, n=DEFAULT_BUFFER_SIZE): + """ + :type fp: asio.interfaces.posix.PosixFile + :type n: int + :rtype: str + """ + r, w, x = select.select([fp.fd], [], [], 5) + + if r: + return os.read(fp.fd, n) + + return None + + @classmethod + def close(cls, fp): + """ + :type fp: asio.interfaces.posix.PosixFile + """ + os.close(fp.fd) + + +class PosixFile(File): + platform_handler = PosixInterface + + def __init__(self, fd, *args, **kwargs): + """ + :type fd: asio.file.File + """ + super(PosixFile, self).__init__(*args, **kwargs) + + self.fd = fd + + def __str__(self): + return "<asio_posix.PosixFile file: %s>" % self.fd diff --git a/Contents/Libraries/Shared/asio/interfaces/windows/__init__.py b/Contents/Libraries/Shared/asio/interfaces/windows/__init__.py new file mode 100644 index 000000000..20ce0bdc2 --- /dev/null +++ b/Contents/Libraries/Shared/asio/interfaces/windows/__init__.py @@ -0,0 +1,201 @@ +# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from asio.file import File, DEFAULT_BUFFER_SIZE +from asio.interfaces.base import Interface + +import os + + +NULL = 0 + +if os.name == 'nt': + from asio.interfaces.windows.interop import WindowsInterop + + +class WindowsInterface(Interface): + @classmethod + def open(cls, file_path, parameters=None): + """ + :type file_path: str + :rtype: asio.interfaces.windows.WindowsFile + """ + if not parameters: + parameters = {} + + return WindowsFile(WindowsInterop.create_file( + file_path, + parameters.get('desired_access', WindowsInterface.GenericAccess.READ), + parameters.get('share_mode', WindowsInterface.ShareMode.ALL), + parameters.get('creation_disposition', WindowsInterface.CreationDisposition.OPEN_EXISTING), + parameters.get('flags_and_attributes', NULL) + )) + + @classmethod + def get_size(cls, fp): + """ + :type fp: asio.interfaces.windows.WindowsFile + :rtype: int + """ + return WindowsInterop.get_file_size(fp.handle) + + @classmethod + def get_path(cls, fp): + """ + :type fp: asio.interfaces.windows.WindowsFile + :rtype: str + """ + + if not fp.file_map: + fp.file_map = WindowsInterop.create_file_mapping(fp.handle, WindowsInterface.Protection.READONLY) + + if not fp.map_view: + fp.map_view = WindowsInterop.map_view_of_file(fp.file_map, WindowsInterface.FileMapAccess.READ, 1) + + file_name = WindowsInterop.get_mapped_file_name(fp.map_view) + + return file_name + + @classmethod + def seek(cls, fp, offset, origin): + """ + :type fp: asio.interfaces.windows.WindowsFile + :type offset: int + :type origin: int + :rtype: int + """ + + return WindowsInterop.set_file_pointer( + fp.handle, + offset, + origin + ) + + @classmethod + def read(cls, fp, n=DEFAULT_BUFFER_SIZE): + """ + :type fp: asio.interfaces.windows.WindowsFile + :type n: int + :rtype: str + """ + return WindowsInterop.read(fp.handle, n) + + @classmethod + def read_into(cls, fp, b): + """ + :type fp: asio.interfaces.windows.WindowsFile + :type b: str + :rtype: int + """ + return WindowsInterop.read_into(fp.handle, b) + + @classmethod + def close(cls, fp): + """ + :type fp: asio.interfaces.windows.WindowsFile + :rtype: bool + """ + if fp.map_view: + WindowsInterop.unmap_view_of_file(fp.map_view) + + if fp.file_map: + WindowsInterop.close_handle(fp.file_map) + + return bool(WindowsInterop.close_handle(fp.handle)) + + class GenericAccess(object): + READ = 0x80000000 + WRITE = 0x40000000 + EXECUTE = 0x20000000 + ALL = 0x10000000 + + class ShareMode(object): + READ = 0x00000001 + WRITE = 0x00000002 + DELETE = 0x00000004 + ALL = READ | WRITE | DELETE + + class CreationDisposition(object): + CREATE_NEW = 1 + CREATE_ALWAYS = 2 + OPEN_EXISTING = 3 + OPEN_ALWAYS = 4 + TRUNCATE_EXISTING = 5 + + class Attribute(object): + READONLY = 0x00000001 + HIDDEN = 0x00000002 + SYSTEM = 0x00000004 + DIRECTORY = 0x00000010 + ARCHIVE = 0x00000020 + DEVICE = 0x00000040 + NORMAL = 0x00000080 + TEMPORARY = 0x00000100 + SPARSE_FILE = 0x00000200 + REPARSE_POINT = 0x00000400 + COMPRESSED = 0x00000800 + OFFLINE = 0x00001000 + NOT_CONTENT_INDEXED = 0x00002000 + ENCRYPTED = 0x00004000 + + class Flag(object): + WRITE_THROUGH = 0x80000000 + OVERLAPPED = 0x40000000 + NO_BUFFERING = 0x20000000 + RANDOM_ACCESS = 0x10000000 + SEQUENTIAL_SCAN = 0x08000000 + DELETE_ON_CLOSE = 0x04000000 + BACKUP_SEMANTICS = 0x02000000 + POSIX_SEMANTICS = 0x01000000 + OPEN_REPARSE_POINT = 0x00200000 + OPEN_NO_RECALL = 0x00100000 + FIRST_PIPE_INSTANCE = 0x00080000 + + class Protection(object): + NOACCESS = 0x01 + READONLY = 0x02 + READWRITE = 0x04 + WRITECOPY = 0x08 + EXECUTE = 0x10 + EXECUTE_READ = 0x20, + EXECUTE_READWRITE = 0x40 + EXECUTE_WRITECOPY = 0x80 + GUARD = 0x100 + NOCACHE = 0x200 + WRITECOMBINE = 0x400 + + class FileMapAccess(object): + COPY = 0x0001 + WRITE = 0x0002 + READ = 0x0004 + ALL_ACCESS = 0x001f + EXECUTE = 0x0020 + + +class WindowsFile(File): + platform_handler = WindowsInterface + + def __init__(self, handle, *args, **kwargs): + super(WindowsFile, self).__init__(*args, **kwargs) + + self.handle = handle + + self.file_map = None + self.map_view = None + + def readinto(self, b): + return self.get_handler().read_into(self, b) + + def __str__(self): + return "<asio_windows.WindowsFile file: %s>" % self.handle diff --git a/Contents/Libraries/Shared/asio/interfaces/windows/interop.py b/Contents/Libraries/Shared/asio/interfaces/windows/interop.py new file mode 100644 index 000000000..7bce197c2 --- /dev/null +++ b/Contents/Libraries/Shared/asio/interfaces/windows/interop.py @@ -0,0 +1,230 @@ +# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ctypes.wintypes import * +from ctypes import * +import logging + +log = logging.getLogger(__name__) + + +CreateFileW = windll.kernel32.CreateFileW +CreateFileW.argtypes = (LPCWSTR, DWORD, DWORD, c_void_p, DWORD, DWORD, HANDLE) +CreateFileW.restype = HANDLE + +ReadFile = windll.kernel32.ReadFile +ReadFile.argtypes = (HANDLE, c_void_p, DWORD, POINTER(DWORD), HANDLE) +ReadFile.restype = BOOL + + +NULL = 0 +MAX_PATH = 260 +DEFAULT_BUFFER_SIZE = 4096 +LPSECURITY_ATTRIBUTES = c_void_p + + +class WindowsInterop(object): + ri_buffer = None + + @classmethod + def create_file(cls, path, desired_access, share_mode, creation_disposition, flags_and_attributes): + h = CreateFileW( + path, + desired_access, + share_mode, + NULL, + creation_disposition, + flags_and_attributes, + NULL + ) + + error = GetLastError() + if error != 0: + raise Exception('[WindowsASIO.open] "%s"' % FormatError(error)) + + return h + + @classmethod + def read(cls, handle, buf_size=DEFAULT_BUFFER_SIZE): + buf = create_string_buffer(buf_size) + bytes_read = c_ulong(0) + + success = ReadFile(handle, buf, buf_size, byref(bytes_read), NULL) + + error = GetLastError() + if error: + log.debug('read_file - error: (%s) "%s"', error, FormatError(error)) + + if not success and error: + raise Exception('[WindowsInterop.read_file] (%s) "%s"' % (error, FormatError(error))) + + # Return if we have a valid buffer + if success and bytes_read.value: + return buf.value + + return None + + @classmethod + def read_into(cls, handle, b): + if cls.ri_buffer is None or len(cls.ri_buffer) < len(b): + cls.ri_buffer = create_string_buffer(len(b)) + + bytes_read = c_ulong(0) + + success = ReadFile(handle, cls.ri_buffer, len(b), byref(bytes_read), NULL) + bytes_read = int(bytes_read.value) + + b[:bytes_read] = cls.ri_buffer[:bytes_read] + + error = GetLastError() + + if not success and error: + raise Exception('[WindowsInterop.read_file] (%s) "%s"' % (error, FormatError(error))) + + # Return if we have a valid buffer + if success and bytes_read: + return bytes_read + + return None + + @classmethod + def set_file_pointer(cls, handle, distance, method): + pos_high = DWORD(NULL) + + result = windll.kernel32.SetFilePointer( + handle, + c_ulong(distance), + byref(pos_high), + DWORD(method) + ) + + if result == -1: + raise Exception('[WindowsASIO.seek] INVALID_SET_FILE_POINTER: "%s"' % FormatError(GetLastError())) + + return result + + @classmethod + def get_file_size(cls, handle): + return windll.kernel32.GetFileSize( + handle, + DWORD(NULL) + ) + + @classmethod + def close_handle(cls, handle): + return windll.kernel32.CloseHandle(handle) + + @classmethod + def create_file_mapping(cls, handle, protect, maximum_size_high=0, maximum_size_low=1): + return HANDLE(windll.kernel32.CreateFileMappingW( + handle, + LPSECURITY_ATTRIBUTES(NULL), + DWORD(protect), + DWORD(maximum_size_high), + DWORD(maximum_size_low), + LPCSTR(NULL) + )) + + @classmethod + def map_view_of_file(cls, map_handle, desired_access, num_bytes, file_offset_high=0, file_offset_low=0): + return HANDLE(windll.kernel32.MapViewOfFile( + map_handle, + DWORD(desired_access), + DWORD(file_offset_high), + DWORD(file_offset_low), + num_bytes + )) + + @classmethod + def unmap_view_of_file(cls, view_handle): + return windll.kernel32.UnmapViewOfFile(view_handle) + + @classmethod + def get_mapped_file_name(cls, view_handle, translate_device_name=True): + buf = create_string_buffer(MAX_PATH + 1) + + result = windll.psapi.GetMappedFileNameW( + cls.get_current_process(), + view_handle, + buf, + MAX_PATH + ) + + # Raise exception on error + error = GetLastError() + if result == 0: + raise Exception(FormatError(error)) + + # Retrieve a clean file name (skipping over NUL bytes) + file_name = cls.clean_buffer_value(buf) + + # If we are not translating the device name return here + if not translate_device_name: + return file_name + + drives = cls.get_logical_drive_strings() + + # Find the drive matching the file_name device name + translated = False + for drive in drives: + device_name = cls.query_dos_device(drive) + + if file_name.startswith(device_name): + file_name = drive + file_name[len(device_name):] + translated = True + break + + if not translated: + raise Exception('Unable to translate device name') + + return file_name + + @classmethod + def get_logical_drive_strings(cls, buf_size=512): + buf = create_string_buffer(buf_size) + + result = windll.kernel32.GetLogicalDriveStringsW(buf_size, buf) + + error = GetLastError() + if result == 0: + raise Exception(FormatError(error)) + + drive_strings = cls.clean_buffer_value(buf) + return [dr for dr in drive_strings.split('\\') if dr != ''] + + @classmethod + def query_dos_device(cls, drive, buf_size=MAX_PATH): + buf = create_string_buffer(buf_size) + + result = windll.kernel32.QueryDosDeviceA( + drive, + buf, + buf_size + ) + + return cls.clean_buffer_value(buf) + + @classmethod + def get_current_process(cls): + return HANDLE(windll.kernel32.GetCurrentProcess()) + + @classmethod + def clean_buffer_value(cls, buf): + value = "" + + for ch in buf.raw: + if ord(ch) != 0: + value += ch + + return value diff --git a/Contents/Libraries/Shared/asio/open_parameters.py b/Contents/Libraries/Shared/asio/open_parameters.py new file mode 100644 index 000000000..a1463854d --- /dev/null +++ b/Contents/Libraries/Shared/asio/open_parameters.py @@ -0,0 +1,47 @@ +from asio.interfaces.posix import PosixInterface +from asio.interfaces.windows import WindowsInterface + + +class OpenParameters(object): + def __init__(self): + self.handlers = {} + + # Update handler_parameters with defaults + self.posix() + self.windows() + + def posix(self, mode=None, buffering=None): + """ + :type mode: str + :type buffering: int + """ + self.handlers.update({PosixInterface: { + 'mode': mode, + 'buffering': buffering + }}) + + def windows(self, desired_access=WindowsInterface.GenericAccess.READ, + share_mode=WindowsInterface.ShareMode.ALL, + creation_disposition=WindowsInterface.CreationDisposition.OPEN_EXISTING, + flags_and_attributes=0): + + """ + :param desired_access: WindowsInterface.DesiredAccess + :type desired_access: int + + :param share_mode: WindowsInterface.ShareMode + :type share_mode: int + + :param creation_disposition: WindowsInterface.CreationDisposition + :type creation_disposition: int + + :param flags_and_attributes: WindowsInterface.Attribute, WindowsInterface.Flag + :type flags_and_attributes: int + """ + + self.handlers.update({WindowsInterface: { + 'desired_access': desired_access, + 'share_mode': share_mode, + 'creation_disposition': creation_disposition, + 'flags_and_attributes': flags_and_attributes + }}) diff --git a/Contents/Libraries/Shared/babelfish/HISTORY.rst b/Contents/Libraries/Shared/babelfish/HISTORY.rst deleted file mode 100644 index 2c8e2abf8..000000000 --- a/Contents/Libraries/Shared/babelfish/HISTORY.rst +++ /dev/null @@ -1,113 +0,0 @@ -Changelog -========= - -0.5.3 ------ -**release date:** 2014-06-22 - -* Better equality semantics for Language, Country, Script - -0.5.2 ------ -**release date:** 2014-05-25 - -* Babelfish objects (Language, Country, Script) are now picklable -* Added support for Python 3.4 - - -0.5.1 ------ -**release date:** 2014-01-26 - -* Add a register method to ConverterManager to register without loading - - -0.5.0 ------ -**release date:** 2014-01-25 - -**WARNING:** Backward incompatible changes - -* Simplify converter management with ConverterManager class -* Make babelfish usable in place -* Add Python 2.6 / 3.2 compatibility - - -0.4.0 ------ -**release date:** 2013-11-21 - -**WARNING:** Backward incompatible changes - -* Add converter support for Country -* Language/country reverse name detection is now case-insensitive -* Add alpha3t, scope and type converters -* Use lazy loading of converters - - -0.3.0 ------ -**release date:** 2013-11-09 - -* Add support for scripts -* Improve built-in converters -* Add support for ietf - - -0.2.1 ------ -**release date:** 2013-11-03 - -* Fix reading of data files - - -0.2.0 ------ -**release date:** 2013-10-31 - -* Add str method -* More explicit exceptions -* Change repr format to use ascii only - - -0.1.5 ------ -**release date:** 2013-10-21 - -* Add a fromcode method on Language class -* Add a codes attribute on converters - - -0.1.4 ------ -**release date:** 2013-10-20 - -* Fix converters not raising NoConversionError - - -0.1.3 ------ -**release date:** 2013-09-29 - -* Fix source distribution - - -0.1.2 ------ -**release date:** 2013-09-29 - -* Add missing files to source distribution - - -0.1.1 ------ -**release date:** 2013-09-28 - -* Fix python3 support - - -0.1 ---- -**release date:** 2013-09-28 - -* Initial version diff --git a/Contents/Libraries/Shared/babelfish/LICENSE b/Contents/Libraries/Shared/babelfish/LICENSE deleted file mode 100644 index a944c46a3..000000000 --- a/Contents/Libraries/Shared/babelfish/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -Copyright (c) 2013, by the respective authors (see AUTHORS file). -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of the BabelFish authors nor the names of its contributors - may be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Contents/Libraries/Shared/babelfish/README.rst b/Contents/Libraries/Shared/babelfish/README.rst deleted file mode 100644 index fecab01c3..000000000 --- a/Contents/Libraries/Shared/babelfish/README.rst +++ /dev/null @@ -1,16 +0,0 @@ -BabelFish -========= - -BabelFish is a Python library to work with countries and languages. - -.. image:: https://travis-ci.org/Diaoul/babelfish.png?branch=master - :target: https://travis-ci.org/Diaoul/babelfish - -.. image:: https://coveralls.io/repos/Diaoul/babelfish/badge.png - :target: https://coveralls.io/r/Diaoul/babelfish - -License -------- - -BabelFish is licensed under the `3-clause BSD license <http://opensource.org/licenses/BSD-3-Clause>`_. -Copyright (c) 2013, the BabelFish authors and contributors. diff --git a/Contents/Libraries/Shared/babelfish/__init__.py b/Contents/Libraries/Shared/babelfish/__init__.py index 26678ce02..559705a25 100644 --- a/Contents/Libraries/Shared/babelfish/__init__.py +++ b/Contents/Libraries/Shared/babelfish/__init__.py @@ -5,10 +5,10 @@ # that can be found in the LICENSE file. # __title__ = 'babelfish' -__version__ = '0.5.3' +__version__ = '0.5.5-dev' __author__ = 'Antoine Bertin' __license__ = 'BSD' -__copyright__ = 'Copyright 2013 the BabelFish authors' +__copyright__ = 'Copyright 2015 the BabelFish authors' import sys diff --git a/Contents/Libraries/Shared/babelfish/converters/__init__.py b/Contents/Libraries/Shared/babelfish/converters/__init__.py index 9a0a1bd90..feb687b0e 100644 --- a/Contents/Libraries/Shared/babelfish/converters/__init__.py +++ b/Contents/Libraries/Shared/babelfish/converters/__init__.py @@ -241,7 +241,14 @@ def __getitem__(self, name): return self.converters[ep.name] for ep in (EntryPoint.parse(c) for c in self.registered_converters + self.internal_converters): if ep.name == name: - self.converters[ep.name] = ep.load(require=False)() + # `require` argument of ep.load() is deprecated in newer versions of setuptools + if hasattr(ep, 'resolve'): + plugin = ep.resolve() + elif hasattr(ep, '_load'): + plugin = ep._load() + else: + plugin = ep.load(require=False) + self.converters[ep.name] = plugin() return self.converters[ep.name] raise KeyError(name) diff --git a/Contents/Libraries/Shared/babelfish/converters/opensubtitles.py b/Contents/Libraries/Shared/babelfish/converters/opensubtitles.py index 101c40fda..5b18e648c 100644 --- a/Contents/Libraries/Shared/babelfish/converters/opensubtitles.py +++ b/Contents/Libraries/Shared/babelfish/converters/opensubtitles.py @@ -17,7 +17,7 @@ def __init__(self): self.to_opensubtitles = {('por', 'BR'): 'pob', ('gre', None): 'ell', ('srp', None): 'scc', ('srp', 'ME'): 'mne'} self.from_opensubtitles = CaseInsensitiveDict({'pob': ('por', 'BR'), 'pb': ('por', 'BR'), 'ell': ('ell', None), 'scc': ('srp', None), 'mne': ('srp', 'ME')}) - self.codes = (self.alpha2_converter.codes | self.alpha3b_converter.codes | set(['pob', 'pb', 'scc', 'mne'])) + self.codes = (self.alpha2_converter.codes | self.alpha3b_converter.codes | set(self.from_opensubtitles.keys())) def convert(self, alpha3, country=None, script=None): alpha3b = self.alpha3b_converter.convert(alpha3, country, script) diff --git a/Contents/Libraries/Shared/babelfish/country.py b/Contents/Libraries/Shared/babelfish/country.py index ce32d9b50..dbc14ce61 100644 --- a/Contents/Libraries/Shared/babelfish/country.py +++ b/Contents/Libraries/Shared/babelfish/country.py @@ -4,7 +4,6 @@ # Use of this source code is governed by the 3-clause BSD license # that can be found in the LICENSE file. # -from __future__ import unicode_literals from collections import namedtuple from functools import partial from pkg_resources import resource_stream # @UnresolvedImport @@ -82,7 +81,10 @@ def __setstate__(self, state): self.alpha2 = state def __getattr__(self, name): - return country_converters[name].convert(self.alpha2) + try: + return country_converters[name].convert(self.alpha2) + except KeyError: + raise AttributeError(name) def __hash__(self): return hash(self.alpha2) diff --git a/Contents/Libraries/Shared/babelfish/data/get_files.py b/Contents/Libraries/Shared/babelfish/data/get_files.py new file mode 100644 index 000000000..aaa090ccc --- /dev/null +++ b/Contents/Libraries/Shared/babelfish/data/get_files.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2013 the BabelFish authors. All rights reserved. +# Use of this source code is governed by the 3-clause BSD license +# that can be found in the LICENSE file. +# +from __future__ import unicode_literals +import os.path +import tempfile +import zipfile +import requests + + +DATA_DIR = os.path.dirname(__file__) + +# iso-3166-1.txt +print('Downloading ISO-3166-1 standard (ISO country codes)...') +with open(os.path.join(DATA_DIR, 'iso-3166-1.txt'), 'w') as f: + r = requests.get('http://www.iso.org/iso/home/standards/country_codes/country_names_and_code_elements_txt.htm') + f.write(r.content.strip()) + +# iso-639-3.tab +print('Downloading ISO-639-3 standard (ISO language codes)...') +with tempfile.TemporaryFile() as f: + r = requests.get('http://www-01.sil.org/iso639-3/iso-639-3_Code_Tables_20130531.zip') + f.write(r.content) + with zipfile.ZipFile(f) as z: + z.extract('iso-639-3.tab', DATA_DIR) + +# iso-15924 +print('Downloading ISO-15924 standard (ISO script codes)...') +with tempfile.TemporaryFile() as f: + r = requests.get('http://www.unicode.org/iso15924/iso15924.txt.zip') + f.write(r.content) + with zipfile.ZipFile(f) as z: + z.extract('iso15924-utf8-20131012.txt', DATA_DIR) + +# opensubtitles supported languages +print('Downloading OpenSubtitles supported languages...') +with open(os.path.join(DATA_DIR, 'opensubtitles_languages.txt'), 'w') as f: + r = requests.get('http://www.opensubtitles.org/addons/export_languages.php') + f.write(r.content) + +print('Done!') diff --git a/Contents/Libraries/Shared/babelfish/data/iso-3166-1.txt b/Contents/Libraries/Shared/babelfish/data/iso-3166-1.txt index da1050722..02b0f467b 100644 --- a/Contents/Libraries/Shared/babelfish/data/iso-3166-1.txt +++ b/Contents/Libraries/Shared/babelfish/data/iso-3166-1.txt @@ -1,250 +1,250 @@ -Country Name;ISO 3166-1-alpha-2 code -AFGHANISTAN;AF -ÅLAND ISLANDS;AX -ALBANIA;AL -ALGERIA;DZ -AMERICAN SAMOA;AS -ANDORRA;AD -ANGOLA;AO -ANGUILLA;AI -ANTARCTICA;AQ -ANTIGUA AND BARBUDA;AG -ARGENTINA;AR -ARMENIA;AM -ARUBA;AW -AUSTRALIA;AU -AUSTRIA;AT -AZERBAIJAN;AZ -BAHAMAS;BS -BAHRAIN;BH -BANGLADESH;BD -BARBADOS;BB -BELARUS;BY -BELGIUM;BE -BELIZE;BZ -BENIN;BJ -BERMUDA;BM -BHUTAN;BT -BOLIVIA, PLURINATIONAL STATE OF;BO -BONAIRE, SINT EUSTATIUS AND SABA;BQ -BOSNIA AND HERZEGOVINA;BA -BOTSWANA;BW -BOUVET ISLAND;BV -BRAZIL;BR -BRITISH INDIAN OCEAN TERRITORY;IO -BRUNEI DARUSSALAM;BN -BULGARIA;BG -BURKINA FASO;BF -BURUNDI;BI -CAMBODIA;KH -CAMEROON;CM -CANADA;CA -CAPE VERDE;CV -CAYMAN ISLANDS;KY -CENTRAL AFRICAN REPUBLIC;CF -CHAD;TD -CHILE;CL -CHINA;CN -CHRISTMAS ISLAND;CX -COCOS (KEELING) ISLANDS;CC -COLOMBIA;CO -COMOROS;KM -CONGO;CG -CONGO, THE DEMOCRATIC REPUBLIC OF THE;CD -COOK ISLANDS;CK -COSTA RICA;CR -CÔTE D'IVOIRE;CI -CROATIA;HR -CUBA;CU -CURAÇAO;CW -CYPRUS;CY -CZECH REPUBLIC;CZ -DENMARK;DK -DJIBOUTI;DJ -DOMINICA;DM -DOMINICAN REPUBLIC;DO -ECUADOR;EC -EGYPT;EG -EL SALVADOR;SV -EQUATORIAL GUINEA;GQ -ERITREA;ER -ESTONIA;EE -ETHIOPIA;ET -FALKLAND ISLANDS (MALVINAS);FK -FAROE ISLANDS;FO -FIJI;FJ -FINLAND;FI -FRANCE;FR -FRENCH GUIANA;GF -FRENCH POLYNESIA;PF -FRENCH SOUTHERN TERRITORIES;TF -GABON;GA -GAMBIA;GM -GEORGIA;GE -GERMANY;DE -GHANA;GH -GIBRALTAR;GI -GREECE;GR -GREENLAND;GL -GRENADA;GD -GUADELOUPE;GP -GUAM;GU -GUATEMALA;GT -GUERNSEY;GG -GUINEA;GN -GUINEA-BISSAU;GW -GUYANA;GY -HAITI;HT -HEARD ISLAND AND MCDONALD ISLANDS;HM -HOLY SEE (VATICAN CITY STATE);VA -HONDURAS;HN -HONG KONG;HK -HUNGARY;HU -ICELAND;IS -INDIA;IN -INDONESIA;ID -IRAN, ISLAMIC REPUBLIC OF;IR -IRAQ;IQ -IRELAND;IE -ISLE OF MAN;IM -ISRAEL;IL -ITALY;IT -JAMAICA;JM -JAPAN;JP -JERSEY;JE -JORDAN;JO -KAZAKHSTAN;KZ -KENYA;KE -KIRIBATI;KI -KOREA, DEMOCRATIC PEOPLE'S REPUBLIC OF;KP -KOREA, REPUBLIC OF;KR -KUWAIT;KW -KYRGYZSTAN;KG -LAO PEOPLE'S DEMOCRATIC REPUBLIC;LA -LATVIA;LV -LEBANON;LB -LESOTHO;LS -LIBERIA;LR -LIBYA;LY -LIECHTENSTEIN;LI -LITHUANIA;LT -LUXEMBOURG;LU -MACAO;MO -MACEDONIA, THE FORMER YUGOSLAV REPUBLIC OF;MK -MADAGASCAR;MG -MALAWI;MW -MALAYSIA;MY -MALDIVES;MV -MALI;ML -MALTA;MT -MARSHALL ISLANDS;MH -MARTINIQUE;MQ -MAURITANIA;MR -MAURITIUS;MU -MAYOTTE;YT -MEXICO;MX -MICRONESIA, FEDERATED STATES OF;FM -MOLDOVA, REPUBLIC OF;MD -MONACO;MC -MONGOLIA;MN -MONTENEGRO;ME -MONTSERRAT;MS -MOROCCO;MA -MOZAMBIQUE;MZ -MYANMAR;MM -NAMIBIA;NA -NAURU;NR -NEPAL;NP -NETHERLANDS;NL -NEW CALEDONIA;NC -NEW ZEALAND;NZ -NICARAGUA;NI -NIGER;NE -NIGERIA;NG -NIUE;NU -NORFOLK ISLAND;NF -NORTHERN MARIANA ISLANDS;MP -NORWAY;NO -OMAN;OM -PAKISTAN;PK -PALAU;PW -PALESTINE, STATE OF;PS -PANAMA;PA -PAPUA NEW GUINEA;PG -PARAGUAY;PY -PERU;PE -PHILIPPINES;PH -PITCAIRN;PN -POLAND;PL -PORTUGAL;PT -PUERTO RICO;PR -QATAR;QA -RÉUNION;RE -ROMANIA;RO -RUSSIAN FEDERATION;RU -RWANDA;RW -SAINT BARTHÉLEMY;BL -SAINT HELENA, ASCENSION AND TRISTAN DA CUNHA;SH -SAINT KITTS AND NEVIS;KN -SAINT LUCIA;LC -SAINT MARTIN (FRENCH PART);MF -SAINT PIERRE AND MIQUELON;PM -SAINT VINCENT AND THE GRENADINES;VC -SAMOA;WS -SAN MARINO;SM -SAO TOME AND PRINCIPE;ST -SAUDI ARABIA;SA -SENEGAL;SN -SERBIA;RS -SEYCHELLES;SC -SIERRA LEONE;SL -SINGAPORE;SG -SINT MAARTEN (DUTCH PART);SX -SLOVAKIA;SK -SLOVENIA;SI -SOLOMON ISLANDS;SB -SOMALIA;SO -SOUTH AFRICA;ZA -SOUTH GEORGIA AND THE SOUTH SANDWICH ISLANDS;GS -SOUTH SUDAN;SS -SPAIN;ES -SRI LANKA;LK -SUDAN;SD -SURINAME;SR -SVALBARD AND JAN MAYEN;SJ -SWAZILAND;SZ -SWEDEN;SE -SWITZERLAND;CH -SYRIAN ARAB REPUBLIC;SY -TAIWAN, PROVINCE OF CHINA;TW -TAJIKISTAN;TJ -TANZANIA, UNITED REPUBLIC OF;TZ -THAILAND;TH -TIMOR-LESTE;TL -TOGO;TG -TOKELAU;TK -TONGA;TO -TRINIDAD AND TOBAGO;TT -TUNISIA;TN -TURKEY;TR -TURKMENISTAN;TM -TURKS AND CAICOS ISLANDS;TC -TUVALU;TV -UGANDA;UG -UKRAINE;UA -UNITED ARAB EMIRATES;AE -UNITED KINGDOM;GB -UNITED STATES;US -UNITED STATES MINOR OUTLYING ISLANDS;UM -URUGUAY;UY -UZBEKISTAN;UZ -VANUATU;VU -VENEZUELA, BOLIVARIAN REPUBLIC OF;VE -VIET NAM;VN -VIRGIN ISLANDS, BRITISH;VG -VIRGIN ISLANDS, U.S.;VI -WALLIS AND FUTUNA;WF -WESTERN SAHARA;EH -YEMEN;YE -ZAMBIA;ZM +Country Name;ISO 3166-1-alpha-2 code +AFGHANISTAN;AF +ÅLAND ISLANDS;AX +ALBANIA;AL +ALGERIA;DZ +AMERICAN SAMOA;AS +ANDORRA;AD +ANGOLA;AO +ANGUILLA;AI +ANTARCTICA;AQ +ANTIGUA AND BARBUDA;AG +ARGENTINA;AR +ARMENIA;AM +ARUBA;AW +AUSTRALIA;AU +AUSTRIA;AT +AZERBAIJAN;AZ +BAHAMAS;BS +BAHRAIN;BH +BANGLADESH;BD +BARBADOS;BB +BELARUS;BY +BELGIUM;BE +BELIZE;BZ +BENIN;BJ +BERMUDA;BM +BHUTAN;BT +BOLIVIA, PLURINATIONAL STATE OF;BO +BONAIRE, SINT EUSTATIUS AND SABA;BQ +BOSNIA AND HERZEGOVINA;BA +BOTSWANA;BW +BOUVET ISLAND;BV +BRAZIL;BR +BRITISH INDIAN OCEAN TERRITORY;IO +BRUNEI DARUSSALAM;BN +BULGARIA;BG +BURKINA FASO;BF +BURUNDI;BI +CAMBODIA;KH +CAMEROON;CM +CANADA;CA +CAPE VERDE;CV +CAYMAN ISLANDS;KY +CENTRAL AFRICAN REPUBLIC;CF +CHAD;TD +CHILE;CL +CHINA;CN +CHRISTMAS ISLAND;CX +COCOS (KEELING) ISLANDS;CC +COLOMBIA;CO +COMOROS;KM +CONGO;CG +CONGO, THE DEMOCRATIC REPUBLIC OF THE;CD +COOK ISLANDS;CK +COSTA RICA;CR +CÔTE D'IVOIRE;CI +CROATIA;HR +CUBA;CU +CURAÇAO;CW +CYPRUS;CY +CZECH REPUBLIC;CZ +DENMARK;DK +DJIBOUTI;DJ +DOMINICA;DM +DOMINICAN REPUBLIC;DO +ECUADOR;EC +EGYPT;EG +EL SALVADOR;SV +EQUATORIAL GUINEA;GQ +ERITREA;ER +ESTONIA;EE +ETHIOPIA;ET +FALKLAND ISLANDS (MALVINAS);FK +FAROE ISLANDS;FO +FIJI;FJ +FINLAND;FI +FRANCE;FR +FRENCH GUIANA;GF +FRENCH POLYNESIA;PF +FRENCH SOUTHERN TERRITORIES;TF +GABON;GA +GAMBIA;GM +GEORGIA;GE +GERMANY;DE +GHANA;GH +GIBRALTAR;GI +GREECE;GR +GREENLAND;GL +GRENADA;GD +GUADELOUPE;GP +GUAM;GU +GUATEMALA;GT +GUERNSEY;GG +GUINEA;GN +GUINEA-BISSAU;GW +GUYANA;GY +HAITI;HT +HEARD ISLAND AND MCDONALD ISLANDS;HM +HOLY SEE (VATICAN CITY STATE);VA +HONDURAS;HN +HONG KONG;HK +HUNGARY;HU +ICELAND;IS +INDIA;IN +INDONESIA;ID +IRAN, ISLAMIC REPUBLIC OF;IR +IRAQ;IQ +IRELAND;IE +ISLE OF MAN;IM +ISRAEL;IL +ITALY;IT +JAMAICA;JM +JAPAN;JP +JERSEY;JE +JORDAN;JO +KAZAKHSTAN;KZ +KENYA;KE +KIRIBATI;KI +KOREA, DEMOCRATIC PEOPLE'S REPUBLIC OF;KP +KOREA, REPUBLIC OF;KR +KUWAIT;KW +KYRGYZSTAN;KG +LAO PEOPLE'S DEMOCRATIC REPUBLIC;LA +LATVIA;LV +LEBANON;LB +LESOTHO;LS +LIBERIA;LR +LIBYA;LY +LIECHTENSTEIN;LI +LITHUANIA;LT +LUXEMBOURG;LU +MACAO;MO +MACEDONIA, THE FORMER YUGOSLAV REPUBLIC OF;MK +MADAGASCAR;MG +MALAWI;MW +MALAYSIA;MY +MALDIVES;MV +MALI;ML +MALTA;MT +MARSHALL ISLANDS;MH +MARTINIQUE;MQ +MAURITANIA;MR +MAURITIUS;MU +MAYOTTE;YT +MEXICO;MX +MICRONESIA, FEDERATED STATES OF;FM +MOLDOVA, REPUBLIC OF;MD +MONACO;MC +MONGOLIA;MN +MONTENEGRO;ME +MONTSERRAT;MS +MOROCCO;MA +MOZAMBIQUE;MZ +MYANMAR;MM +NAMIBIA;NA +NAURU;NR +NEPAL;NP +NETHERLANDS;NL +NEW CALEDONIA;NC +NEW ZEALAND;NZ +NICARAGUA;NI +NIGER;NE +NIGERIA;NG +NIUE;NU +NORFOLK ISLAND;NF +NORTHERN MARIANA ISLANDS;MP +NORWAY;NO +OMAN;OM +PAKISTAN;PK +PALAU;PW +PALESTINE, STATE OF;PS +PANAMA;PA +PAPUA NEW GUINEA;PG +PARAGUAY;PY +PERU;PE +PHILIPPINES;PH +PITCAIRN;PN +POLAND;PL +PORTUGAL;PT +PUERTO RICO;PR +QATAR;QA +RÉUNION;RE +ROMANIA;RO +RUSSIAN FEDERATION;RU +RWANDA;RW +SAINT BARTHÉLEMY;BL +SAINT HELENA, ASCENSION AND TRISTAN DA CUNHA;SH +SAINT KITTS AND NEVIS;KN +SAINT LUCIA;LC +SAINT MARTIN (FRENCH PART);MF +SAINT PIERRE AND MIQUELON;PM +SAINT VINCENT AND THE GRENADINES;VC +SAMOA;WS +SAN MARINO;SM +SAO TOME AND PRINCIPE;ST +SAUDI ARABIA;SA +SENEGAL;SN +SERBIA;RS +SEYCHELLES;SC +SIERRA LEONE;SL +SINGAPORE;SG +SINT MAARTEN (DUTCH PART);SX +SLOVAKIA;SK +SLOVENIA;SI +SOLOMON ISLANDS;SB +SOMALIA;SO +SOUTH AFRICA;ZA +SOUTH GEORGIA AND THE SOUTH SANDWICH ISLANDS;GS +SOUTH SUDAN;SS +SPAIN;ES +SRI LANKA;LK +SUDAN;SD +SURINAME;SR +SVALBARD AND JAN MAYEN;SJ +SWAZILAND;SZ +SWEDEN;SE +SWITZERLAND;CH +SYRIAN ARAB REPUBLIC;SY +TAIWAN, PROVINCE OF CHINA;TW +TAJIKISTAN;TJ +TANZANIA, UNITED REPUBLIC OF;TZ +THAILAND;TH +TIMOR-LESTE;TL +TOGO;TG +TOKELAU;TK +TONGA;TO +TRINIDAD AND TOBAGO;TT +TUNISIA;TN +TURKEY;TR +TURKMENISTAN;TM +TURKS AND CAICOS ISLANDS;TC +TUVALU;TV +UGANDA;UG +UKRAINE;UA +UNITED ARAB EMIRATES;AE +UNITED KINGDOM;GB +UNITED STATES;US +UNITED STATES MINOR OUTLYING ISLANDS;UM +URUGUAY;UY +UZBEKISTAN;UZ +VANUATU;VU +VENEZUELA, BOLIVARIAN REPUBLIC OF;VE +VIET NAM;VN +VIRGIN ISLANDS, BRITISH;VG +VIRGIN ISLANDS, U.S.;VI +WALLIS AND FUTUNA;WF +WESTERN SAHARA;EH +YEMEN;YE +ZAMBIA;ZM ZIMBABWE;ZW \ No newline at end of file diff --git a/Contents/Libraries/Shared/babelfish/data/iso-639-3.tab b/Contents/Libraries/Shared/babelfish/data/iso-639-3.tab index f66d683be..68c48dcb9 100644 --- a/Contents/Libraries/Shared/babelfish/data/iso-639-3.tab +++ b/Contents/Libraries/Shared/babelfish/data/iso-639-3.tab @@ -1,7875 +1,7875 @@ -Id Part2B Part2T Part1 Scope Language_Type Ref_Name Comment -aaa I L Ghotuo -aab I L Alumu-Tesu -aac I L Ari -aad I L Amal -aae I L Arbëreshë Albanian -aaf I L Aranadan -aag I L Ambrak -aah I L Abu' Arapesh -aai I L Arifama-Miniafia -aak I L Ankave -aal I L Afade -aam I L Aramanik -aan I L Anambé -aao I L Algerian Saharan Arabic -aap I L Pará Arára -aaq I E Eastern Abnaki -aar aar aar aa I L Afar -aas I L Aasáx -aat I L Arvanitika Albanian -aau I L Abau -aaw I L Solong -aax I L Mandobo Atas -aaz I L Amarasi -aba I L Abé -abb I L Bankon -abc I L Ambala Ayta -abd I L Manide -abe I E Western Abnaki -abf I L Abai Sungai -abg I L Abaga -abh I L Tajiki Arabic -abi I L Abidji -abj I E Aka-Bea -abk abk abk ab I L Abkhazian -abl I L Lampung Nyo -abm I L Abanyom -abn I L Abua -abo I L Abon -abp I L Abellen Ayta -abq I L Abaza -abr I L Abron -abs I L Ambonese Malay -abt I L Ambulas -abu I L Abure -abv I L Baharna Arabic -abw I L Pal -abx I L Inabaknon -aby I L Aneme Wake -abz I L Abui -aca I L Achagua -acb I L Áncá -acd I L Gikyode -ace ace ace I L Achinese -acf I L Saint Lucian Creole French -ach ach ach I L Acoli -aci I E Aka-Cari -ack I E Aka-Kora -acl I E Akar-Bale -acm I L Mesopotamian Arabic -acn I L Achang -acp I L Eastern Acipa -acq I L Ta'izzi-Adeni Arabic -acr I L Achi -acs I E Acroá -act I L Achterhoeks -acu I L Achuar-Shiwiar -acv I L Achumawi -acw I L Hijazi Arabic -acx I L Omani Arabic -acy I L Cypriot Arabic -acz I L Acheron -ada ada ada I L Adangme -adb I L Adabe -add I L Dzodinka -ade I L Adele -adf I L Dhofari Arabic -adg I L Andegerebinha -adh I L Adhola -adi I L Adi -adj I L Adioukrou -adl I L Galo -adn I L Adang -ado I L Abu -adp I L Adap -adq I L Adangbe -adr I L Adonara -ads I L Adamorobe Sign Language -adt I L Adnyamathanha -adu I L Aduge -adw I L Amundava -adx I L Amdo Tibetan -ady ady ady I L Adyghe -adz I L Adzera -aea I E Areba -aeb I L Tunisian Arabic -aec I L Saidi Arabic -aed I L Argentine Sign Language -aee I L Northeast Pashayi -aek I L Haeke -ael I L Ambele -aem I L Arem -aen I L Armenian Sign Language -aeq I L Aer -aer I L Eastern Arrernte -aes I E Alsea -aeu I L Akeu -aew I L Ambakich -aey I L Amele -aez I L Aeka -afb I L Gulf Arabic -afd I L Andai -afe I L Putukwam -afg I L Afghan Sign Language -afh afh afh I C Afrihili -afi I L Akrukay -afk I L Nanubae -afn I L Defaka -afo I L Eloyi -afp I L Tapei -afr afr afr af I L Afrikaans -afs I L Afro-Seminole Creole -aft I L Afitti -afu I L Awutu -afz I L Obokuitai -aga I E Aguano -agb I L Legbo -agc I L Agatu -agd I L Agarabi -age I L Angal -agf I L Arguni -agg I L Angor -agh I L Ngelima -agi I L Agariya -agj I L Argobba -agk I L Isarog Agta -agl I L Fembe -agm I L Angaataha -agn I L Agutaynen -ago I L Tainae -agq I L Aghem -agr I L Aguaruna -ags I L Esimbi -agt I L Central Cagayan Agta -agu I L Aguacateco -agv I L Remontado Dumagat -agw I L Kahua -agx I L Aghul -agy I L Southern Alta -agz I L Mt. Iriga Agta -aha I L Ahanta -ahb I L Axamb -ahg I L Qimant -ahh I L Aghu -ahi I L Tiagbamrin Aizi -ahk I L Akha -ahl I L Igo -ahm I L Mobumrin Aizi -ahn I L Àhàn -aho I E Ahom -ahp I L Aproumu Aizi -ahr I L Ahirani -ahs I L Ashe -aht I L Ahtena -aia I L Arosi -aib I L Ainu (China) -aic I L Ainbai -aid I E Alngith -aie I L Amara -aif I L Agi -aig I L Antigua and Barbuda Creole English -aih I L Ai-Cham -aii I L Assyrian Neo-Aramaic -aij I L Lishanid Noshan -aik I L Ake -ail I L Aimele -aim I L Aimol -ain ain ain I L Ainu (Japan) -aio I L Aiton -aip I L Burumakok -aiq I L Aimaq -air I L Airoran -ais I L Nataoran Amis -ait I E Arikem -aiw I L Aari -aix I L Aighon -aiy I L Ali -aja I L Aja (Sudan) -ajg I L Aja (Benin) -aji I L Ajië -ajn I L Andajin -ajp I L South Levantine Arabic -ajt I L Judeo-Tunisian Arabic -aju I L Judeo-Moroccan Arabic -ajw I E Ajawa -ajz I L Amri Karbi -aka aka aka ak M L Akan -akb I L Batak Angkola -akc I L Mpur -akd I L Ukpet-Ehom -ake I L Akawaio -akf I L Akpa -akg I L Anakalangu -akh I L Angal Heneng -aki I L Aiome -akj I E Aka-Jeru -akk akk akk I A Akkadian -akl I L Aklanon -akm I E Aka-Bo -ako I L Akurio -akp I L Siwu -akq I L Ak -akr I L Araki -aks I L Akaselem -akt I L Akolet -aku I L Akum -akv I L Akhvakh -akw I L Akwa -akx I E Aka-Kede -aky I E Aka-Kol -akz I L Alabama -ala I L Alago -alc I L Qawasqar -ald I L Alladian -ale ale ale I L Aleut -alf I L Alege -alh I L Alawa -ali I L Amaimon -alj I L Alangan -alk I L Alak -all I L Allar -alm I L Amblong -aln I L Gheg Albanian -alo I L Larike-Wakasihu -alp I L Alune -alq I L Algonquin -alr I L Alutor -als I L Tosk Albanian -alt alt alt I L Southern Altai -alu I L 'Are'are -alw I L Alaba-K’abeena -alx I L Amol -aly I L Alyawarr -alz I L Alur -ama I E Amanayé -amb I L Ambo -amc I L Amahuaca -ame I L Yanesha' -amf I L Hamer-Banna -amg I L Amurdak -amh amh amh am I L Amharic -ami I L Amis -amj I L Amdang -amk I L Ambai -aml I L War-Jaintia -amm I L Ama (Papua New Guinea) -amn I L Amanab -amo I L Amo -amp I L Alamblak -amq I L Amahai -amr I L Amarakaeri -ams I L Southern Amami-Oshima -amt I L Amto -amu I L Guerrero Amuzgo -amv I L Ambelau -amw I L Western Neo-Aramaic -amx I L Anmatyerre -amy I L Ami -amz I E Atampaya -ana I E Andaqui -anb I E Andoa -anc I L Ngas -and I L Ansus -ane I L Xârâcùù -anf I L Animere -ang ang ang I H Old English (ca. 450-1100) -anh I L Nend -ani I L Andi -anj I L Anor -ank I L Goemai -anl I L Anu-Hkongso Chin -anm I L Anal -ann I L Obolo -ano I L Andoque -anp anp anp I L Angika -anq I L Jarawa (India) -anr I L Andh -ans I E Anserma -ant I L Antakarinya -anu I L Anuak -anv I L Denya -anw I L Anaang -anx I L Andra-Hus -any I L Anyin -anz I L Anem -aoa I L Angolar -aob I L Abom -aoc I L Pemon -aod I L Andarum -aoe I L Angal Enen -aof I L Bragat -aog I L Angoram -aoh I E Arma -aoi I L Anindilyakwa -aoj I L Mufian -aok I L Arhö -aol I L Alor -aom I L Ömie -aon I L Bumbita Arapesh -aor I E Aore -aos I L Taikat -aot I L A'tong -aou I L A'ou -aox I L Atorada -aoz I L Uab Meto -apb I L Sa'a -apc I L North Levantine Arabic -apd I L Sudanese Arabic -ape I L Bukiyip -apf I L Pahanan Agta -apg I L Ampanang -aph I L Athpariya -api I L Apiaká -apj I L Jicarilla Apache -apk I L Kiowa Apache -apl I L Lipan Apache -apm I L Mescalero-Chiricahua Apache -apn I L Apinayé -apo I L Ambul -app I L Apma -apq I L A-Pucikwar -apr I L Arop-Lokep -aps I L Arop-Sissano -apt I L Apatani -apu I L Apurinã -apv I E Alapmunte -apw I L Western Apache -apx I L Aputai -apy I L Apalaí -apz I L Safeyoka -aqc I L Archi -aqd I L Ampari Dogon -aqg I L Arigidi -aqm I L Atohwaim -aqn I L Northern Alta -aqp I E Atakapa -aqr I L Arhâ -aqz I L Akuntsu -ara ara ara ar M L Arabic -arb I L Standard Arabic -arc arc arc I A Official Aramaic (700-300 BCE) -ard I E Arabana -are I L Western Arrarnta -arg arg arg an I L Aragonese -arh I L Arhuaco -ari I L Arikara -arj I E Arapaso -ark I L Arikapú -arl I L Arabela -arn arn arn I L Mapudungun -aro I L Araona -arp arp arp I L Arapaho -arq I L Algerian Arabic -arr I L Karo (Brazil) -ars I L Najdi Arabic -aru I E Aruá (Amazonas State) -arv I L Arbore -arw arw arw I L Arawak -arx I L Aruá (Rodonia State) -ary I L Moroccan Arabic -arz I L Egyptian Arabic -asa I L Asu (Tanzania) -asb I L Assiniboine -asc I L Casuarina Coast Asmat -asd I L Asas -ase I L American Sign Language -asf I L Australian Sign Language -asg I L Cishingini -ash I E Abishira -asi I L Buruwai -asj I L Sari -ask I L Ashkun -asl I L Asilulu -asm asm asm as I L Assamese -asn I L Xingú Asuriní -aso I L Dano -asp I L Algerian Sign Language -asq I L Austrian Sign Language -asr I L Asuri -ass I L Ipulo -ast ast ast I L Asturian -asu I L Tocantins Asurini -asv I L Asoa -asw I L Australian Aborigines Sign Language -asx I L Muratayak -asy I L Yaosakor Asmat -asz I L As -ata I L Pele-Ata -atb I L Zaiwa -atc I E Atsahuaca -atd I L Ata Manobo -ate I L Atemble -atg I L Ivbie North-Okpela-Arhe -ati I L Attié -atj I L Atikamekw -atk I L Ati -atl I L Mt. Iraya Agta -atm I L Ata -atn I L Ashtiani -ato I L Atong -atp I L Pudtol Atta -atq I L Aralle-Tabulahan -atr I L Waimiri-Atroari -ats I L Gros Ventre -att I L Pamplona Atta -atu I L Reel -atv I L Northern Altai -atw I L Atsugewi -atx I L Arutani -aty I L Aneityum -atz I L Arta -aua I L Asumboa -aub I L Alugu -auc I L Waorani -aud I L Anuta -aue I L =/Kx'au//'ein -aug I L Aguna -auh I L Aushi -aui I L Anuki -auj I L Awjilah -auk I L Heyo -aul I L Aulua -aum I L Asu (Nigeria) -aun I L Molmo One -auo I E Auyokawa -aup I L Makayam -auq I L Anus -aur I L Aruek -aut I L Austral -auu I L Auye -auw I L Awyi -aux I E Aurá -auy I L Awiyaana -auz I L Uzbeki Arabic -ava ava ava av I L Avaric -avb I L Avau -avd I L Alviri-Vidari -ave ave ave ae I A Avestan -avi I L Avikam -avk I C Kotava -avl I L Eastern Egyptian Bedawi Arabic -avm I E Angkamuthi -avn I L Avatime -avo I E Agavotaguerra -avs I E Aushiri -avt I L Au -avu I L Avokaya -avv I L Avá-Canoeiro -awa awa awa I L Awadhi -awb I L Awa (Papua New Guinea) -awc I L Cicipu -awe I L Awetí -awg I E Anguthimri -awh I L Awbono -awi I L Aekyom -awk I E Awabakal -awm I L Arawum -awn I L Awngi -awo I L Awak -awr I L Awera -aws I L South Awyu -awt I L Araweté -awu I L Central Awyu -awv I L Jair Awyu -aww I L Awun -awx I L Awara -awy I L Edera Awyu -axb I E Abipon -axe I E Ayerrerenge -axg I E Mato Grosso Arára -axk I L Yaka (Central African Republic) -axl I E Lower Southern Aranda -axm I H Middle Armenian -axx I L Xârâgurè -aya I L Awar -ayb I L Ayizo Gbe -ayc I L Southern Aymara -ayd I E Ayabadhu -aye I L Ayere -ayg I L Ginyanga -ayh I L Hadrami Arabic -ayi I L Leyigha -ayk I L Akuku -ayl I L Libyan Arabic -aym aym aym ay M L Aymara -ayn I L Sanaani Arabic -ayo I L Ayoreo -ayp I L North Mesopotamian Arabic -ayq I L Ayi (Papua New Guinea) -ayr I L Central Aymara -ays I L Sorsogon Ayta -ayt I L Magbukun Ayta -ayu I L Ayu -ayy I E Tayabas Ayta -ayz I L Mai Brat -aza I L Azha -azb I L South Azerbaijani -azd I L Eastern Durango Nahuatl -aze aze aze az M L Azerbaijani -azg I L San Pedro Amuzgos Amuzgo -azj I L North Azerbaijani -azm I L Ipalapa Amuzgo -azn I L Western Durango Nahuatl -azo I L Awing -azt I L Faire Atta -azz I L Highland Puebla Nahuatl -baa I L Babatana -bab I L Bainouk-Gunyuño -bac I L Badui -bae I E Baré -baf I L Nubaca -bag I L Tuki -bah I L Bahamas Creole English -baj I L Barakai -bak bak bak ba I L Bashkir -bal bal bal M L Baluchi -bam bam bam bm I L Bambara -ban ban ban I L Balinese -bao I L Waimaha -bap I L Bantawa -bar I L Bavarian -bas bas bas I L Basa (Cameroon) -bau I L Bada (Nigeria) -bav I L Vengo -baw I L Bambili-Bambui -bax I L Bamun -bay I L Batuley -bba I L Baatonum -bbb I L Barai -bbc I L Batak Toba -bbd I L Bau -bbe I L Bangba -bbf I L Baibai -bbg I L Barama -bbh I L Bugan -bbi I L Barombi -bbj I L Ghomálá' -bbk I L Babanki -bbl I L Bats -bbm I L Babango -bbn I L Uneapa -bbo I L Northern Bobo Madaré -bbp I L West Central Banda -bbq I L Bamali -bbr I L Girawa -bbs I L Bakpinka -bbt I L Mburku -bbu I L Kulung (Nigeria) -bbv I L Karnai -bbw I L Baba -bbx I L Bubia -bby I L Befang -bbz I L Babalia Creole Arabic -bca I L Central Bai -bcb I L Bainouk-Samik -bcc I L Southern Balochi -bcd I L North Babar -bce I L Bamenyam -bcf I L Bamu -bcg I L Baga Binari -bch I L Bariai -bci I L Baoulé -bcj I L Bardi -bck I L Bunaba -bcl I L Central Bikol -bcm I L Bannoni -bcn I L Bali (Nigeria) -bco I L Kaluli -bcp I L Bali (Democratic Republic of Congo) -bcq I L Bench -bcr I L Babine -bcs I L Kohumono -bct I L Bendi -bcu I L Awad Bing -bcv I L Shoo-Minda-Nye -bcw I L Bana -bcy I L Bacama -bcz I L Bainouk-Gunyaamolo -bda I L Bayot -bdb I L Basap -bdc I L Emberá-Baudó -bdd I L Bunama -bde I L Bade -bdf I L Biage -bdg I L Bonggi -bdh I L Baka (Sudan) -bdi I L Burun -bdj I L Bai -bdk I L Budukh -bdl I L Indonesian Bajau -bdm I L Buduma -bdn I L Baldemu -bdo I L Morom -bdp I L Bende -bdq I L Bahnar -bdr I L West Coast Bajau -bds I L Burunge -bdt I L Bokoto -bdu I L Oroko -bdv I L Bodo Parja -bdw I L Baham -bdx I L Budong-Budong -bdy I L Bandjalang -bdz I L Badeshi -bea I L Beaver -beb I L Bebele -bec I L Iceve-Maci -bed I L Bedoanas -bee I L Byangsi -bef I L Benabena -beg I L Belait -beh I L Biali -bei I L Bekati' -bej bej bej I L Beja -bek I L Bebeli -bel bel bel be I L Belarusian -bem bem bem I L Bemba (Zambia) -ben ben ben bn I L Bengali -beo I L Beami -bep I L Besoa -beq I L Beembe -bes I L Besme -bet I L Guiberoua Béte -beu I L Blagar -bev I L Daloa Bété -bew I L Betawi -bex I L Jur Modo -bey I L Beli (Papua New Guinea) -bez I L Bena (Tanzania) -bfa I L Bari -bfb I L Pauri Bareli -bfc I L Northern Bai -bfd I L Bafut -bfe I L Betaf -bff I L Bofi -bfg I L Busang Kayan -bfh I L Blafe -bfi I L British Sign Language -bfj I L Bafanji -bfk I L Ban Khor Sign Language -bfl I L Banda-Ndélé -bfm I L Mmen -bfn I L Bunak -bfo I L Malba Birifor -bfp I L Beba -bfq I L Badaga -bfr I L Bazigar -bfs I L Southern Bai -bft I L Balti -bfu I L Gahri -bfw I L Bondo -bfx I L Bantayanon -bfy I L Bagheli -bfz I L Mahasu Pahari -bga I L Gwamhi-Wuri -bgb I L Bobongko -bgc I L Haryanvi -bgd I L Rathwi Bareli -bge I L Bauria -bgf I L Bangandu -bgg I L Bugun -bgi I L Giangan -bgj I L Bangolan -bgk I L Bit -bgl I L Bo (Laos) -bgm I L Baga Mboteni -bgn I L Western Balochi -bgo I L Baga Koga -bgp I L Eastern Balochi -bgq I L Bagri -bgr I L Bawm Chin -bgs I L Tagabawa -bgt I L Bughotu -bgu I L Mbongno -bgv I L Warkay-Bipim -bgw I L Bhatri -bgx I L Balkan Gagauz Turkish -bgy I L Benggoi -bgz I L Banggai -bha I L Bharia -bhb I L Bhili -bhc I L Biga -bhd I L Bhadrawahi -bhe I L Bhaya -bhf I L Odiai -bhg I L Binandere -bhh I L Bukharic -bhi I L Bhilali -bhj I L Bahing -bhl I L Bimin -bhm I L Bathari -bhn I L Bohtan Neo-Aramaic -bho bho bho I L Bhojpuri -bhp I L Bima -bhq I L Tukang Besi South -bhr I L Bara Malagasy -bhs I L Buwal -bht I L Bhattiyali -bhu I L Bhunjia -bhv I L Bahau -bhw I L Biak -bhx I L Bhalay -bhy I L Bhele -bhz I L Bada (Indonesia) -bia I L Badimaya -bib I L Bissa -bic I L Bikaru -bid I L Bidiyo -bie I L Bepour -bif I L Biafada -big I L Biangai -bij I L Vaghat-Ya-Bijim-Legeri -bik bik bik M L Bikol -bil I L Bile -bim I L Bimoba -bin bin bin I L Bini -bio I L Nai -bip I L Bila -biq I L Bipi -bir I L Bisorio -bis bis bis bi I L Bislama -bit I L Berinomo -biu I L Biete -biv I L Southern Birifor -biw I L Kol (Cameroon) -bix I L Bijori -biy I L Birhor -biz I L Baloi -bja I L Budza -bjb I E Banggarla -bjc I L Bariji -bje I L Biao-Jiao Mien -bjf I L Barzani Jewish Neo-Aramaic -bjg I L Bidyogo -bjh I L Bahinemo -bji I L Burji -bjj I L Kanauji -bjk I L Barok -bjl I L Bulu (Papua New Guinea) -bjm I L Bajelani -bjn I L Banjar -bjo I L Mid-Southern Banda -bjp I L Fanamaket -bjr I L Binumarien -bjs I L Bajan -bjt I L Balanta-Ganja -bju I L Busuu -bjv I L Bedjond -bjw I L Bakwé -bjx I L Banao Itneg -bjy I E Bayali -bjz I L Baruga -bka I L Kyak -bkc I L Baka (Cameroon) -bkd I L Binukid -bkf I L Beeke -bkg I L Buraka -bkh I L Bakoko -bki I L Baki -bkj I L Pande -bkk I L Brokskat -bkl I L Berik -bkm I L Kom (Cameroon) -bkn I L Bukitan -bko I L Kwa' -bkp I L Boko (Democratic Republic of Congo) -bkq I L Bakairí -bkr I L Bakumpai -bks I L Northern Sorsoganon -bkt I L Boloki -bku I L Buhid -bkv I L Bekwarra -bkw I L Bekwel -bkx I L Baikeno -bky I L Bokyi -bkz I L Bungku -bla bla bla I L Siksika -blb I L Bilua -blc I L Bella Coola -bld I L Bolango -ble I L Balanta-Kentohe -blf I L Buol -blg I L Balau -blh I L Kuwaa -bli I L Bolia -blj I L Bolongan -blk I L Pa'o Karen -bll I E Biloxi -blm I L Beli (Sudan) -bln I L Southern Catanduanes Bikol -blo I L Anii -blp I L Blablanga -blq I L Baluan-Pam -blr I L Blang -bls I L Balaesang -blt I L Tai Dam -blv I L Bolo -blw I L Balangao -blx I L Mag-Indi Ayta -bly I L Notre -blz I L Balantak -bma I L Lame -bmb I L Bembe -bmc I L Biem -bmd I L Baga Manduri -bme I L Limassa -bmf I L Bom -bmg I L Bamwe -bmh I L Kein -bmi I L Bagirmi -bmj I L Bote-Majhi -bmk I L Ghayavi -bml I L Bomboli -bmm I L Northern Betsimisaraka Malagasy -bmn I E Bina (Papua New Guinea) -bmo I L Bambalang -bmp I L Bulgebi -bmq I L Bomu -bmr I L Muinane -bms I L Bilma Kanuri -bmt I L Biao Mon -bmu I L Somba-Siawari -bmv I L Bum -bmw I L Bomwali -bmx I L Baimak -bmy I L Bemba (Democratic Republic of Congo) -bmz I L Baramu -bna I L Bonerate -bnb I L Bookan -bnc M L Bontok -bnd I L Banda (Indonesia) -bne I L Bintauna -bnf I L Masiwang -bng I L Benga -bni I L Bangi -bnj I L Eastern Tawbuid -bnk I L Bierebo -bnl I L Boon -bnm I L Batanga -bnn I L Bunun -bno I L Bantoanon -bnp I L Bola -bnq I L Bantik -bnr I L Butmas-Tur -bns I L Bundeli -bnu I L Bentong -bnv I L Bonerif -bnw I L Bisis -bnx I L Bangubangu -bny I L Bintulu -bnz I L Beezen -boa I L Bora -bob I L Aweer -bod tib bod bo I L Tibetan -boe I L Mundabli -bof I L Bolon -bog I L Bamako Sign Language -boh I L Boma -boi I E Barbareño -boj I L Anjam -bok I L Bonjo -bol I L Bole -bom I L Berom -bon I L Bine -boo I L Tiemacèwè Bozo -bop I L Bonkiman -boq I L Bogaya -bor I L Borôro -bos bos bos bs I L Bosnian -bot I L Bongo -bou I L Bondei -bov I L Tuwuli -bow I E Rema -box I L Buamu -boy I L Bodo (Central African Republic) -boz I L Tiéyaxo Bozo -bpa I L Daakaka -bpb I E Barbacoas -bpd I L Banda-Banda -bpg I L Bonggo -bph I L Botlikh -bpi I L Bagupi -bpj I L Binji -bpk I L Orowe -bpl I L Broome Pearling Lugger Pidgin -bpm I L Biyom -bpn I L Dzao Min -bpo I L Anasi -bpp I L Kaure -bpq I L Banda Malay -bpr I L Koronadal Blaan -bps I L Sarangani Blaan -bpt I E Barrow Point -bpu I L Bongu -bpv I L Bian Marind -bpw I L Bo (Papua New Guinea) -bpx I L Palya Bareli -bpy I L Bishnupriya -bpz I L Bilba -bqa I L Tchumbuli -bqb I L Bagusa -bqc I L Boko (Benin) -bqd I L Bung -bqf I E Baga Kaloum -bqg I L Bago-Kusuntu -bqh I L Baima -bqi I L Bakhtiari -bqj I L Bandial -bqk I L Banda-Mbrès -bql I L Bilakura -bqm I L Wumboko -bqn I L Bulgarian Sign Language -bqo I L Balo -bqp I L Busa -bqq I L Biritai -bqr I L Burusu -bqs I L Bosngun -bqt I L Bamukumbit -bqu I L Boguru -bqv I L Koro Wachi -bqw I L Buru (Nigeria) -bqx I L Baangi -bqy I L Bengkala Sign Language -bqz I L Bakaka -bra bra bra I L Braj -brb I L Lave -brc I E Berbice Creole Dutch -brd I L Baraamu -bre bre bre br I L Breton -brf I L Bera -brg I L Baure -brh I L Brahui -bri I L Mokpwe -brj I L Bieria -brk I E Birked -brl I L Birwa -brm I L Barambu -brn I L Boruca -bro I L Brokkat -brp I L Barapasi -brq I L Breri -brr I L Birao -brs I L Baras -brt I L Bitare -bru I L Eastern Bru -brv I L Western Bru -brw I L Bellari -brx I L Bodo (India) -bry I L Burui -brz I L Bilbil -bsa I L Abinomn -bsb I L Brunei Bisaya -bsc I L Bassari -bse I L Wushi -bsf I L Bauchi -bsg I L Bashkardi -bsh I L Kati -bsi I L Bassossi -bsj I L Bangwinji -bsk I L Burushaski -bsl I E Basa-Gumna -bsm I L Busami -bsn I L Barasana-Eduria -bso I L Buso -bsp I L Baga Sitemu -bsq I L Bassa -bsr I L Bassa-Kontagora -bss I L Akoose -bst I L Basketo -bsu I L Bahonsuai -bsv I E Baga Sobané -bsw I L Baiso -bsx I L Yangkam -bsy I L Sabah Bisaya -bta I L Bata -btc I L Bati (Cameroon) -btd I L Batak Dairi -bte I E Gamo-Ningi -btf I L Birgit -btg I L Gagnoa Bété -bth I L Biatah Bidayuh -bti I L Burate -btj I L Bacanese Malay -btl I L Bhatola -btm I L Batak Mandailing -btn I L Ratagnon -bto I L Rinconada Bikol -btp I L Budibud -btq I L Batek -btr I L Baetora -bts I L Batak Simalungun -btt I L Bete-Bendi -btu I L Batu -btv I L Bateri -btw I L Butuanon -btx I L Batak Karo -bty I L Bobot -btz I L Batak Alas-Kluet -bua bua bua M L Buriat -bub I L Bua -buc I L Bushi -bud I L Ntcham -bue I E Beothuk -buf I L Bushoong -bug bug bug I L Buginese -buh I L Younuo Bunu -bui I L Bongili -buj I L Basa-Gurmana -buk I L Bugawac -bul bul bul bg I L Bulgarian -bum I L Bulu (Cameroon) -bun I L Sherbro -buo I L Terei -bup I L Busoa -buq I L Brem -bus I L Bokobaru -but I L Bungain -buu I L Budu -buv I L Bun -buw I L Bubi -bux I L Boghom -buy I L Bullom So -buz I L Bukwen -bva I L Barein -bvb I L Bube -bvc I L Baelelea -bvd I L Baeggu -bve I L Berau Malay -bvf I L Boor -bvg I L Bonkeng -bvh I L Bure -bvi I L Belanda Viri -bvj I L Baan -bvk I L Bukat -bvl I L Bolivian Sign Language -bvm I L Bamunka -bvn I L Buna -bvo I L Bolgo -bvp I L Bumang -bvq I L Birri -bvr I L Burarra -bvt I L Bati (Indonesia) -bvu I L Bukit Malay -bvv I E Baniva -bvw I L Boga -bvx I L Dibole -bvy I L Baybayanon -bvz I L Bauzi -bwa I L Bwatoo -bwb I L Namosi-Naitasiri-Serua -bwc I L Bwile -bwd I L Bwaidoka -bwe I L Bwe Karen -bwf I L Boselewa -bwg I L Barwe -bwh I L Bishuo -bwi I L Baniwa -bwj I L Láá Láá Bwamu -bwk I L Bauwaki -bwl I L Bwela -bwm I L Biwat -bwn I L Wunai Bunu -bwo I L Boro (Ethiopia) -bwp I L Mandobo Bawah -bwq I L Southern Bobo Madaré -bwr I L Bura-Pabir -bws I L Bomboma -bwt I L Bafaw-Balong -bwu I L Buli (Ghana) -bww I L Bwa -bwx I L Bu-Nao Bunu -bwy I L Cwi Bwamu -bwz I L Bwisi -bxa I L Tairaha -bxb I L Belanda Bor -bxc I L Molengue -bxd I L Pela -bxe I L Birale -bxf I L Bilur -bxg I L Bangala -bxh I L Buhutu -bxi I E Pirlatapa -bxj I L Bayungu -bxk I L Bukusu -bxl I L Jalkunan -bxm I L Mongolia Buriat -bxn I L Burduna -bxo I L Barikanchi -bxp I L Bebil -bxq I L Beele -bxr I L Russia Buriat -bxs I L Busam -bxu I L China Buriat -bxv I L Berakou -bxw I L Bankagooma -bxx I L Borna (Democratic Republic of Congo) -bxz I L Binahari -bya I L Batak -byb I L Bikya -byc I L Ubaghara -byd I L Benyadu' -bye I L Pouye -byf I L Bete -byg I E Baygo -byh I L Bhujel -byi I L Buyu -byj I L Bina (Nigeria) -byk I L Biao -byl I L Bayono -bym I L Bidyara -byn byn byn I L Bilin -byo I L Biyo -byp I L Bumaji -byq I E Basay -byr I L Baruya -bys I L Burak -byt I E Berti -byv I L Medumba -byw I L Belhariya -byx I L Qaqet -byy I L Buya -byz I L Banaro -bza I L Bandi -bzb I L Andio -bzc I L Southern Betsimisaraka Malagasy -bzd I L Bribri -bze I L Jenaama Bozo -bzf I L Boikin -bzg I L Babuza -bzh I L Mapos Buang -bzi I L Bisu -bzj I L Belize Kriol English -bzk I L Nicaragua Creole English -bzl I L Boano (Sulawesi) -bzm I L Bolondo -bzn I L Boano (Maluku) -bzo I L Bozaba -bzp I L Kemberano -bzq I L Buli (Indonesia) -bzr I E Biri -bzs I L Brazilian Sign Language -bzt I C Brithenig -bzu I L Burmeso -bzv I L Naami -bzw I L Basa (Nigeria) -bzx I L Kɛlɛngaxo Bozo -bzy I L Obanliku -bzz I L Evant -caa I L Chortí -cab I L Garifuna -cac I L Chuj -cad cad cad I L Caddo -cae I L Lehar -caf I L Southern Carrier -cag I L Nivaclé -cah I L Cahuarano -caj I E Chané -cak I L Kaqchikel -cal I L Carolinian -cam I L Cemuhî -can I L Chambri -cao I L Chácobo -cap I L Chipaya -caq I L Car Nicobarese -car car car I L Galibi Carib -cas I L Tsimané -cat cat cat ca I L Catalan -cav I L Cavineña -caw I L Callawalla -cax I L Chiquitano -cay I L Cayuga -caz I E Canichana -cbb I L Cabiyarí -cbc I L Carapana -cbd I L Carijona -cbe I E Chipiajes -cbg I L Chimila -cbh I E Cagua -cbi I L Chachi -cbj I L Ede Cabe -cbk I L Chavacano -cbl I L Bualkhaw Chin -cbn I L Nyahkur -cbo I L Izora -cbr I L Cashibo-Cacataibo -cbs I L Cashinahua -cbt I L Chayahuita -cbu I L Candoshi-Shapra -cbv I L Cacua -cbw I L Kinabalian -cby I L Carabayo -cca I E Cauca -ccc I L Chamicuro -ccd I L Cafundo Creole -cce I L Chopi -ccg I L Samba Daka -cch I L Atsam -ccj I L Kasanga -ccl I L Cutchi-Swahili -ccm I L Malaccan Creole Malay -cco I L Comaltepec Chinantec -ccp I L Chakma -ccr I E Cacaopera -cda I L Choni -cde I L Chenchu -cdf I L Chiru -cdg I L Chamari -cdh I L Chambeali -cdi I L Chodri -cdj I L Churahi -cdm I L Chepang -cdn I L Chaudangsi -cdo I L Min Dong Chinese -cdr I L Cinda-Regi-Tiyal -cds I L Chadian Sign Language -cdy I L Chadong -cdz I L Koda -cea I E Lower Chehalis -ceb ceb ceb I L Cebuano -ceg I L Chamacoco -cek I L Eastern Khumi Chin -cen I L Cen -ces cze ces cs I L Czech -cet I L Centúúm -cfa I L Dijim-Bwilim -cfd I L Cara -cfg I L Como Karim -cfm I L Falam Chin -cga I L Changriwa -cgc I L Kagayanen -cgg I L Chiga -cgk I L Chocangacakha -cha cha cha ch I L Chamorro -chb chb chb I E Chibcha -chc I E Catawba -chd I L Highland Oaxaca Chontal -che che che ce I L Chechen -chf I L Tabasco Chontal -chg chg chg I E Chagatai -chh I L Chinook -chj I L Ojitlán Chinantec -chk chk chk I L Chuukese -chl I L Cahuilla -chm chm chm M L Mari (Russia) -chn chn chn I L Chinook jargon -cho cho cho I L Choctaw -chp chp chp I L Chipewyan -chq I L Quiotepec Chinantec -chr chr chr I L Cherokee -cht I E Cholón -chu chu chu cu I A Church Slavic -chv chv chv cv I L Chuvash -chw I L Chuwabu -chx I L Chantyal -chy chy chy I L Cheyenne -chz I L Ozumacín Chinantec -cia I L Cia-Cia -cib I L Ci Gbe -cic I L Chickasaw -cid I E Chimariko -cie I L Cineni -cih I L Chinali -cik I L Chitkuli Kinnauri -cim I L Cimbrian -cin I L Cinta Larga -cip I L Chiapanec -cir I L Tiri -ciw I L Chippewa -ciy I L Chaima -cja I L Western Cham -cje I L Chru -cjh I E Upper Chehalis -cji I L Chamalal -cjk I L Chokwe -cjm I L Eastern Cham -cjn I L Chenapian -cjo I L Ashéninka Pajonal -cjp I L Cabécar -cjs I L Shor -cjv I L Chuave -cjy I L Jinyu Chinese -ckb I L Central Kurdish -ckh I L Chak -ckl I L Cibak -ckn I L Kaang Chin -cko I L Anufo -ckq I L Kajakse -ckr I L Kairak -cks I L Tayo -ckt I L Chukot -cku I L Koasati -ckv I L Kavalan -ckx I L Caka -cky I L Cakfem-Mushere -ckz I L Cakchiquel-Quiché Mixed Language -cla I L Ron -clc I L Chilcotin -cld I L Chaldean Neo-Aramaic -cle I L Lealao Chinantec -clh I L Chilisso -cli I L Chakali -clj I L Laitu Chin -clk I L Idu-Mishmi -cll I L Chala -clm I L Clallam -clo I L Lowland Oaxaca Chontal -clt I L Lautu Chin -clu I L Caluyanun -clw I L Chulym -cly I L Eastern Highland Chatino -cma I L Maa -cme I L Cerma -cmg I H Classical Mongolian -cmi I L Emberá-Chamí -cml I L Campalagian -cmm I E Michigamea -cmn I L Mandarin Chinese -cmo I L Central Mnong -cmr I L Mro-Khimi Chin -cms I A Messapic -cmt I L Camtho -cna I L Changthang -cnb I L Chinbon Chin -cnc I L Côông -cng I L Northern Qiang -cnh I L Haka Chin -cni I L Asháninka -cnk I L Khumi Chin -cnl I L Lalana Chinantec -cno I L Con -cns I L Central Asmat -cnt I L Tepetotutla Chinantec -cnu I L Chenoua -cnw I L Ngawn Chin -cnx I H Middle Cornish -coa I L Cocos Islands Malay -cob I E Chicomuceltec -coc I L Cocopa -cod I L Cocama-Cocamilla -coe I L Koreguaje -cof I L Colorado -cog I L Chong -coh I L Chonyi-Dzihana-Kauma -coj I E Cochimi -cok I L Santa Teresa Cora -col I L Columbia-Wenatchi -com I L Comanche -con I L Cofán -coo I L Comox -cop cop cop I E Coptic -coq I E Coquille -cor cor cor kw I L Cornish -cos cos cos co I L Corsican -cot I L Caquinte -cou I L Wamey -cov I L Cao Miao -cow I E Cowlitz -cox I L Nanti -coy I E Coyaima -coz I L Chochotec -cpa I L Palantla Chinantec -cpb I L Ucayali-Yurúa Ashéninka -cpc I L Ajyíninka Apurucayali -cpg I E Cappadocian Greek -cpi I L Chinese Pidgin English -cpn I L Cherepon -cpo I L Kpeego -cps I L Capiznon -cpu I L Pichis Ashéninka -cpx I L Pu-Xian Chinese -cpy I L South Ucayali Ashéninka -cqd I L Chuanqiandian Cluster Miao -cqu I L Chilean Quechua -cra I L Chara -crb I E Island Carib -crc I L Lonwolwol -crd I L Coeur d'Alene -cre cre cre cr M L Cree -crf I E Caramanta -crg I L Michif -crh crh crh I L Crimean Tatar -cri I L Sãotomense -crj I L Southern East Cree -crk I L Plains Cree -crl I L Northern East Cree -crm I L Moose Cree -crn I L El Nayar Cora -cro I L Crow -crq I L Iyo'wujwa Chorote -crr I E Carolina Algonquian -crs I L Seselwa Creole French -crt I L Iyojwa'ja Chorote -crv I L Chaura -crw I L Chrau -crx I L Carrier -cry I L Cori -crz I E Cruzeño -csa I L Chiltepec Chinantec -csb csb csb I L Kashubian -csc I L Catalan Sign Language -csd I L Chiangmai Sign Language -cse I L Czech Sign Language -csf I L Cuba Sign Language -csg I L Chilean Sign Language -csh I L Asho Chin -csi I E Coast Miwok -csj I L Songlai Chin -csk I L Jola-Kasa -csl I L Chinese Sign Language -csm I L Central Sierra Miwok -csn I L Colombian Sign Language -cso I L Sochiapam Chinantec -csq I L Croatia Sign Language -csr I L Costa Rican Sign Language -css I E Southern Ohlone -cst I L Northern Ohlone -csv I L Sumtu Chin -csw I L Swampy Cree -csy I L Siyin Chin -csz I L Coos -cta I L Tataltepec Chatino -ctc I L Chetco -ctd I L Tedim Chin -cte I L Tepinapa Chinantec -ctg I L Chittagonian -cth I L Thaiphum Chin -ctl I L Tlacoatzintepec Chinantec -ctm I E Chitimacha -ctn I L Chhintange -cto I L Emberá-Catío -ctp I L Western Highland Chatino -cts I L Northern Catanduanes Bikol -ctt I L Wayanad Chetti -ctu I L Chol -ctz I L Zacatepec Chatino -cua I L Cua -cub I L Cubeo -cuc I L Usila Chinantec -cug I L Cung -cuh I L Chuka -cui I L Cuiba -cuj I L Mashco Piro -cuk I L San Blas Kuna -cul I L Culina -cum I E Cumeral -cuo I E Cumanagoto -cup I E Cupeño -cuq I L Cun -cur I L Chhulung -cut I L Teutila Cuicatec -cuu I L Tai Ya -cuv I L Cuvok -cuw I L Chukwa -cux I L Tepeuxila Cuicatec -cvg I L Chug -cvn I L Valle Nacional Chinantec -cwa I L Kabwa -cwb I L Maindo -cwd I L Woods Cree -cwe I L Kwere -cwg I L Chewong -cwt I L Kuwaataay -cya I L Nopala Chatino -cyb I E Cayubaba -cym wel cym cy I L Welsh -cyo I L Cuyonon -czh I L Huizhou Chinese -czk I E Knaanic -czn I L Zenzontepec Chatino -czo I L Min Zhong Chinese -czt I L Zotung Chin -daa I L Dangaléat -dac I L Dambi -dad I L Marik -dae I L Duupa -dag I L Dagbani -dah I L Gwahatike -dai I L Day -daj I L Dar Fur Daju -dak dak dak I L Dakota -dal I L Dahalo -dam I L Damakawa -dan dan dan da I L Danish -dao I L Daai Chin -daq I L Dandami Maria -dar dar dar I L Dargwa -das I L Daho-Doo -dau I L Dar Sila Daju -dav I L Taita -daw I L Davawenyo -dax I L Dayi -daz I L Dao -dba I L Bangime -dbb I L Deno -dbd I L Dadiya -dbe I L Dabe -dbf I L Edopi -dbg I L Dogul Dom Dogon -dbi I L Doka -dbj I L Ida'an -dbl I L Dyirbal -dbm I L Duguri -dbn I L Duriankere -dbo I L Dulbu -dbp I L Duwai -dbq I L Daba -dbr I L Dabarre -dbt I L Ben Tey Dogon -dbu I L Bondum Dom Dogon -dbv I L Dungu -dbw I L Bankan Tey Dogon -dby I L Dibiyaso -dcc I L Deccan -dcr I E Negerhollands -dda I E Dadi Dadi -ddd I L Dongotono -dde I L Doondo -ddg I L Fataluku -ddi I L West Goodenough -ddj I L Jaru -ddn I L Dendi (Benin) -ddo I L Dido -ddr I E Dhudhuroa -dds I L Donno So Dogon -ddw I L Dawera-Daweloor -dec I L Dagik -ded I L Dedua -dee I L Dewoin -def I L Dezfuli -deg I L Degema -deh I L Dehwari -dei I L Demisa -dek I L Dek -del del del M L Delaware -dem I L Dem -den den den M L Slave (Athapascan) -dep I E Pidgin Delaware -deq I L Dendi (Central African Republic) -der I L Deori -des I L Desano -deu ger deu de I L German -dev I L Domung -dez I L Dengese -dga I L Southern Dagaare -dgb I L Bunoge Dogon -dgc I L Casiguran Dumagat Agta -dgd I L Dagaari Dioula -dge I L Degenan -dgg I L Doga -dgh I L Dghwede -dgi I L Northern Dagara -dgk I L Dagba -dgl I L Andaandi -dgn I E Dagoman -dgo I L Dogri (individual language) -dgr dgr dgr I L Dogrib -dgs I L Dogoso -dgt I E Ndra'ngith -dgu I L Degaru -dgw I E Daungwurrung -dgx I L Doghoro -dgz I L Daga -dhd I L Dhundari -dhg I L Djangu -dhi I L Dhimal -dhl I L Dhalandji -dhm I L Zemba -dhn I L Dhanki -dho I L Dhodia -dhr I L Dhargari -dhs I L Dhaiso -dhu I E Dhurga -dhv I L Dehu -dhw I L Dhanwar (Nepal) -dhx I L Dhungaloo -dia I L Dia -dib I L South Central Dinka -dic I L Lakota Dida -did I L Didinga -dif I E Dieri -dig I L Digo -dih I L Kumiai -dii I L Dimbong -dij I L Dai -dik I L Southwestern Dinka -dil I L Dilling -dim I L Dime -din din din M L Dinka -dio I L Dibo -dip I L Northeastern Dinka -diq I L Dimli (individual language) -dir I L Dirim -dis I L Dimasa -dit I E Dirari -diu I L Diriku -div div div dv I L Dhivehi -diw I L Northwestern Dinka -dix I L Dixon Reef -diy I L Diuwe -diz I L Ding -dja I E Djadjawurrung -djb I L Djinba -djc I L Dar Daju Daju -djd I L Djamindjung -dje I L Zarma -djf I E Djangun -dji I L Djinang -djj I L Djeebbana -djk I L Eastern Maroon Creole -djm I L Jamsay Dogon -djn I L Djauan -djo I L Jangkang -djr I L Djambarrpuyngu -dju I L Kapriman -djw I E Djawi -dka I L Dakpakha -dkk I L Dakka -dkr I L Kuijau -dks I L Southeastern Dinka -dkx I L Mazagway -dlg I L Dolgan -dlk I L Dahalik -dlm I E Dalmatian -dln I L Darlong -dma I L Duma -dmb I L Mombo Dogon -dmc I L Gavak -dmd I E Madhi Madhi -dme I L Dugwor -dmg I L Upper Kinabatangan -dmk I L Domaaki -dml I L Dameli -dmm I L Dama -dmo I L Kemedzung -dmr I L East Damar -dms I L Dampelas -dmu I L Dubu -dmv I L Dumpas -dmw I L Mudburra -dmx I L Dema -dmy I L Demta -dna I L Upper Grand Valley Dani -dnd I L Daonda -dne I L Ndendeule -dng I L Dungan -dni I L Lower Grand Valley Dani -dnj I L Dan -dnk I L Dengka -dnn I L Dzùùngoo -dnr I L Danaru -dnt I L Mid Grand Valley Dani -dnu I L Danau -dnv I L Danu -dnw I L Western Dani -dny I L Dení -doa I L Dom -dob I L Dobu -doc I L Northern Dong -doe I L Doe -dof I L Domu -doh I L Dong -doi doi doi M L Dogri (macrolanguage) -dok I L Dondo -dol I L Doso -don I L Toura (Papua New Guinea) -doo I L Dongo -dop I L Lukpa -doq I L Dominican Sign Language -dor I L Dori'o -dos I L Dogosé -dot I L Dass -dov I L Dombe -dow I L Doyayo -dox I L Bussa -doy I L Dompo -doz I L Dorze -dpp I L Papar -drb I L Dair -drc I L Minderico -drd I L Darmiya -dre I L Dolpo -drg I L Rungus -dri I L C'lela -drl I L Paakantyi -drn I L West Damar -dro I L Daro-Matu Melanau -drq I E Dura -drr I E Dororo -drs I L Gedeo -drt I L Drents -dru I L Rukai -dry I L Darai -dsb dsb dsb I L Lower Sorbian -dse I L Dutch Sign Language -dsh I L Daasanach -dsi I L Disa -dsl I L Danish Sign Language -dsn I L Dusner -dso I L Desiya -dsq I L Tadaksahak -dta I L Daur -dtb I L Labuk-Kinabatangan Kadazan -dtd I L Ditidaht -dth I E Adithinngithigh -dti I L Ana Tinga Dogon -dtk I L Tene Kan Dogon -dtm I L Tomo Kan Dogon -dto I L Tommo So Dogon -dtp I L Central Dusun -dtr I L Lotud -dts I L Toro So Dogon -dtt I L Toro Tegu Dogon -dtu I L Tebul Ure Dogon -dty I L Dotyali -dua dua dua I L Duala -dub I L Dubli -duc I L Duna -dud I L Hun-Saare -due I L Umiray Dumaget Agta -duf I L Dumbea -dug I L Duruma -duh I L Dungra Bhil -dui I L Dumun -duj I L Dhuwal -duk I L Uyajitaya -dul I L Alabat Island Agta -dum dum dum I H Middle Dutch (ca. 1050-1350) -dun I L Dusun Deyah -duo I L Dupaninan Agta -dup I L Duano -duq I L Dusun Malang -dur I L Dii -dus I L Dumi -duu I L Drung -duv I L Duvle -duw I L Dusun Witu -dux I L Duungooma -duy I E Dicamay Agta -duz I E Duli -dva I L Duau -dwa I L Diri -dwr I L Dawro -dws I C Dutton World Speedwords -dww I L Dawawa -dya I L Dyan -dyb I E Dyaberdyaber -dyd I E Dyugun -dyg I E Villa Viciosa Agta -dyi I L Djimini Senoufo -dym I L Yanda Dom Dogon -dyn I L Dyangadi -dyo I L Jola-Fonyi -dyu dyu dyu I L Dyula -dyy I L Dyaabugay -dza I L Tunzu -dzd I L Daza -dze I E Djiwarli -dzg I L Dazaga -dzl I L Dzalakha -dzn I L Dzando -dzo dzo dzo dz I L Dzongkha -eaa I E Karenggapa -ebg I L Ebughu -ebk I L Eastern Bontok -ebo I L Teke-Ebo -ebr I L Ebrié -ebu I L Embu -ecr I A Eteocretan -ecs I L Ecuadorian Sign Language -ecy I A Eteocypriot -eee I L E -efa I L Efai -efe I L Efe -efi efi efi I L Efik -ega I L Ega -egl I L Emilian -ego I L Eggon -egy egy egy I A Egyptian (Ancient) -ehu I L Ehueun -eip I L Eipomek -eit I L Eitiep -eiv I L Askopan -eja I L Ejamat -eka eka eka I L Ekajuk -ekc I E Eastern Karnic -eke I L Ekit -ekg I L Ekari -eki I L Eki -ekk I L Standard Estonian -ekl I L Kol (Bangladesh) -ekm I L Elip -eko I L Koti -ekp I L Ekpeye -ekr I L Yace -eky I L Eastern Kayah -ele I L Elepi -elh I L El Hugeirat -eli I E Nding -elk I L Elkei -ell gre ell el I L Modern Greek (1453-) -elm I L Eleme -elo I L El Molo -elu I L Elu -elx elx elx I A Elamite -ema I L Emai-Iuleha-Ora -emb I L Embaloh -eme I L Emerillon -emg I L Eastern Meohang -emi I L Mussau-Emira -emk I L Eastern Maninkakan -emm I E Mamulique -emn I L Eman -emo I E Emok -emp I L Northern Emberá -ems I L Pacific Gulf Yupik -emu I L Eastern Muria -emw I L Emplawas -emx I L Erromintxela -emy I E Epigraphic Mayan -ena I L Apali -enb I L Markweeta -enc I L En -end I L Ende -enf I L Forest Enets -eng eng eng en I L English -enh I L Tundra Enets -enm enm enm I H Middle English (1100-1500) -enn I L Engenni -eno I L Enggano -enq I L Enga -enr I L Emumu -enu I L Enu -env I L Enwan (Edu State) -enw I L Enwan (Akwa Ibom State) -eot I L Beti (Côte d'Ivoire) -epi I L Epie -epo epo epo eo I C Esperanto -era I L Eravallan -erg I L Sie -erh I L Eruwa -eri I L Ogea -erk I L South Efate -ero I L Horpa -err I E Erre -ers I L Ersu -ert I L Eritai -erw I L Erokwanas -ese I L Ese Ejja -esh I L Eshtehardi -esi I L North Alaskan Inupiatun -esk I L Northwest Alaska Inupiatun -esl I L Egypt Sign Language -esm I E Esuma -esn I L Salvadoran Sign Language -eso I L Estonian Sign Language -esq I E Esselen -ess I L Central Siberian Yupik -est est est et M L Estonian -esu I L Central Yupik -etb I L Etebi -etc I E Etchemin -eth I L Ethiopian Sign Language -etn I L Eton (Vanuatu) -eto I L Eton (Cameroon) -etr I L Edolo -ets I L Yekhee -ett I A Etruscan -etu I L Ejagham -etx I L Eten -etz I L Semimi -eus baq eus eu I L Basque -eve I L Even -evh I L Uvbie -evn I L Evenki -ewe ewe ewe ee I L Ewe -ewo ewo ewo I L Ewondo -ext I L Extremaduran -eya I E Eyak -eyo I L Keiyo -eza I L Ezaa -eze I L Uzekwe -faa I L Fasu -fab I L Fa d'Ambu -fad I L Wagi -faf I L Fagani -fag I L Finongan -fah I L Baissa Fali -fai I L Faiwol -faj I L Faita -fak I L Fang (Cameroon) -fal I L South Fali -fam I L Fam -fan fan fan I L Fang (Equatorial Guinea) -fao fao fao fo I L Faroese -fap I L Palor -far I L Fataleka -fas per fas fa M L Persian -fat fat fat I L Fanti -fau I L Fayu -fax I L Fala -fay I L Southwestern Fars -faz I L Northwestern Fars -fbl I L West Albay Bikol -fcs I L Quebec Sign Language -fer I L Feroge -ffi I L Foia Foia -ffm I L Maasina Fulfulde -fgr I L Fongoro -fia I L Nobiin -fie I L Fyer -fij fij fij fj I L Fijian -fil fil fil I L Filipino -fin fin fin fi I L Finnish -fip I L Fipa -fir I L Firan -fit I L Tornedalen Finnish -fiw I L Fiwaga -fkk I L Kirya-Konzəl -fkv I L Kven Finnish -fla I L Kalispel-Pend d'Oreille -flh I L Foau -fli I L Fali -fll I L North Fali -fln I E Flinders Island -flr I L Fuliiru -fly I L Tsotsitaal -fmp I L Fe'fe' -fmu I L Far Western Muria -fng I L Fanagalo -fni I L Fania -fod I L Foodo -foi I L Foi -fom I L Foma -fon fon fon I L Fon -for I L Fore -fos I E Siraya -fpe I L Fernando Po Creole English -fqs I L Fas -fra fre fra fr I L French -frc I L Cajun French -frd I L Fordata -frk I E Frankish -frm frm frm I H Middle French (ca. 1400-1600) -fro fro fro I H Old French (842-ca. 1400) -frp I L Arpitan -frq I L Forak -frr frr frr I L Northern Frisian -frs frs frs I L Eastern Frisian -frt I L Fortsenal -fry fry fry fy I L Western Frisian -fse I L Finnish Sign Language -fsl I L French Sign Language -fss I L Finland-Swedish Sign Language -fub I L Adamawa Fulfulde -fuc I L Pulaar -fud I L East Futuna -fue I L Borgu Fulfulde -fuf I L Pular -fuh I L Western Niger Fulfulde -fui I L Bagirmi Fulfulde -fuj I L Ko -ful ful ful ff M L Fulah -fum I L Fum -fun I L Fulniô -fuq I L Central-Eastern Niger Fulfulde -fur fur fur I L Friulian -fut I L Futuna-Aniwa -fuu I L Furu -fuv I L Nigerian Fulfulde -fuy I L Fuyug -fvr I L Fur -fwa I L Fwâi -fwe I L Fwe -gaa gaa gaa I L Ga -gab I L Gabri -gac I L Mixed Great Andamanese -gad I L Gaddang -gae I L Guarequena -gaf I L Gende -gag I L Gagauz -gah I L Alekano -gai I L Borei -gaj I L Gadsup -gak I L Gamkonora -gal I L Galolen -gam I L Kandawo -gan I L Gan Chinese -gao I L Gants -gap I L Gal -gaq I L Gata' -gar I L Galeya -gas I L Adiwasi Garasia -gat I L Kenati -gau I L Mudhili Gadaba -gaw I L Nobonob -gax I L Borana-Arsi-Guji Oromo -gay gay gay I L Gayo -gaz I L West Central Oromo -gba gba gba M L Gbaya (Central African Republic) -gbb I L Kaytetye -gbd I L Karadjeri -gbe I L Niksek -gbf I L Gaikundi -gbg I L Gbanziri -gbh I L Defi Gbe -gbi I L Galela -gbj I L Bodo Gadaba -gbk I L Gaddi -gbl I L Gamit -gbm I L Garhwali -gbn I L Mo'da -gbo I L Northern Grebo -gbp I L Gbaya-Bossangoa -gbq I L Gbaya-Bozoum -gbr I L Gbagyi -gbs I L Gbesi Gbe -gbu I L Gagadu -gbv I L Gbanu -gbw I L Gabi-Gabi -gbx I L Eastern Xwla Gbe -gby I L Gbari -gbz I L Zoroastrian Dari -gcc I L Mali -gcd I E Ganggalida -gce I E Galice -gcf I L Guadeloupean Creole French -gcl I L Grenadian Creole English -gcn I L Gaina -gcr I L Guianese Creole French -gct I L Colonia Tovar German -gda I L Gade Lohar -gdb I L Pottangi Ollar Gadaba -gdc I E Gugu Badhun -gdd I L Gedaged -gde I L Gude -gdf I L Guduf-Gava -gdg I L Ga'dang -gdh I L Gadjerawang -gdi I L Gundi -gdj I L Gurdjar -gdk I L Gadang -gdl I L Dirasha -gdm I L Laal -gdn I L Umanakaina -gdo I L Ghodoberi -gdq I L Mehri -gdr I L Wipi -gds I L Ghandruk Sign Language -gdt I E Kungardutyi -gdu I L Gudu -gdx I L Godwari -gea I L Geruma -geb I L Kire -gec I L Gboloo Grebo -ged I L Gade -geg I L Gengle -geh I L Hutterite German -gei I L Gebe -gej I L Gen -gek I L Yiwom -gel I L ut-Ma'in -geq I L Geme -ges I L Geser-Gorom -gew I L Gera -gex I L Garre -gey I L Enya -gez gez gez I A Geez -gfk I L Patpatar -gft I E Gafat -gfx I L Mangetti Dune !Xung -gga I L Gao -ggb I L Gbii -ggd I E Gugadj -gge I L Guragone -ggg I L Gurgula -ggk I E Kungarakany -ggl I L Ganglau -ggm I E Gugu Mini -ggn I L Eastern Gurung -ggo I L Southern Gondi -ggt I L Gitua -ggu I L Gagu -ggw I L Gogodala -gha I L Ghadamès -ghc I E Hiberno-Scottish Gaelic -ghe I L Southern Ghale -ghh I L Northern Ghale -ghk I L Geko Karen -ghl I L Ghulfan -ghn I L Ghanongga -gho I E Ghomara -ghr I L Ghera -ghs I L Guhu-Samane -ght I L Kuke -gia I L Kitja -gib I L Gibanawa -gic I L Gail -gid I L Gidar -gig I L Goaria -gih I L Githabul -gil gil gil I L Gilbertese -gim I L Gimi (Eastern Highlands) -gin I L Hinukh -gip I L Gimi (West New Britain) -giq I L Green Gelao -gir I L Red Gelao -gis I L North Giziga -git I L Gitxsan -giu I L Mulao -giw I L White Gelao -gix I L Gilima -giy I L Giyug -giz I L South Giziga -gji I L Geji -gjk I L Kachi Koli -gjm I E Gunditjmara -gjn I L Gonja -gju I L Gujari -gka I L Guya -gke I L Ndai -gkn I L Gokana -gko I E Kok-Nar -gkp I L Guinea Kpelle -gla gla gla gd I L Scottish Gaelic -glc I L Bon Gula -gld I L Nanai -gle gle gle ga I L Irish -glg glg glg gl I L Galician -glh I L Northwest Pashayi -gli I E Guliguli -glj I L Gula Iro -glk I L Gilaki -gll I E Garlali -glo I L Galambu -glr I L Glaro-Twabo -glu I L Gula (Chad) -glv glv glv gv I L Manx -glw I L Glavda -gly I E Gule -gma I E Gambera -gmb I L Gula'alaa -gmd I L Mághdì -gmh gmh gmh I H Middle High German (ca. 1050-1500) -gml I H Middle Low German -gmm I L Gbaya-Mbodomo -gmn I L Gimnime -gmu I L Gumalu -gmv I L Gamo -gmx I L Magoma -gmy I A Mycenaean Greek -gmz I L Mgbolizhia -gna I L Kaansa -gnb I L Gangte -gnc I E Guanche -gnd I L Zulgo-Gemzek -gne I L Ganang -gng I L Ngangam -gnh I L Lere -gni I L Gooniyandi -gnk I L //Gana -gnl I E Gangulu -gnm I L Ginuman -gnn I L Gumatj -gno I L Northern Gondi -gnq I L Gana -gnr I E Gureng Gureng -gnt I L Guntai -gnu I L Gnau -gnw I L Western Bolivian Guaraní -gnz I L Ganzi -goa I L Guro -gob I L Playero -goc I L Gorakor -god I L Godié -goe I L Gongduk -gof I L Gofa -gog I L Gogo -goh goh goh I H Old High German (ca. 750-1050) -goi I L Gobasi -goj I L Gowlan -gok I L Gowli -gol I L Gola -gom I L Goan Konkani -gon gon gon M L Gondi -goo I L Gone Dau -gop I L Yeretuar -goq I L Gorap -gor gor gor I L Gorontalo -gos I L Gronings -got got got I A Gothic -gou I L Gavar -gow I L Gorowa -gox I L Gobu -goy I L Goundo -goz I L Gozarkhani -gpa I L Gupa-Abawa -gpe I L Ghanaian Pidgin English -gpn I L Taiap -gqa I L Ga'anda -gqi I L Guiqiong -gqn I E Guana (Brazil) -gqr I L Gor -gqu I L Qau -gra I L Rajput Garasia -grb grb grb M L Grebo -grc grc grc I H Ancient Greek (to 1453) -grd I L Guruntum-Mbaaru -grg I L Madi -grh I L Gbiri-Niragu -gri I L Ghari -grj I L Southern Grebo -grm I L Kota Marudu Talantang -grn grn grn gn M L Guarani -gro I L Groma -grq I L Gorovu -grr I L Taznatit -grs I L Gresi -grt I L Garo -gru I L Kistane -grv I L Central Grebo -grw I L Gweda -grx I L Guriaso -gry I L Barclayville Grebo -grz I L Guramalum -gse I L Ghanaian Sign Language -gsg I L German Sign Language -gsl I L Gusilay -gsm I L Guatemalan Sign Language -gsn I L Gusan -gso I L Southwest Gbaya -gsp I L Wasembo -gss I L Greek Sign Language -gsw gsw gsw I L Swiss German -gta I L Guató -gti I L Gbati-ri -gtu I E Aghu-Tharnggala -gua I L Shiki -gub I L Guajajára -guc I L Wayuu -gud I L Yocoboué Dida -gue I L Gurinji -guf I L Gupapuyngu -gug I L Paraguayan Guaraní -guh I L Guahibo -gui I L Eastern Bolivian Guaraní -guj guj guj gu I L Gujarati -guk I L Gumuz -gul I L Sea Island Creole English -gum I L Guambiano -gun I L Mbyá Guaraní -guo I L Guayabero -gup I L Gunwinggu -guq I L Aché -gur I L Farefare -gus I L Guinean Sign Language -gut I L Maléku Jaíka -guu I L Yanomamö -guv I E Gey -guw I L Gun -gux I L Gourmanchéma -guz I L Gusii -gva I L Guana (Paraguay) -gvc I L Guanano -gve I L Duwet -gvf I L Golin -gvj I L Guajá -gvl I L Gulay -gvm I L Gurmana -gvn I L Kuku-Yalanji -gvo I L Gavião Do Jiparaná -gvp I L Pará Gavião -gvr I L Western Gurung -gvs I L Gumawana -gvy I E Guyani -gwa I L Mbato -gwb I L Gwa -gwc I L Kalami -gwd I L Gawwada -gwe I L Gweno -gwf I L Gowro -gwg I L Moo -gwi gwi gwi I L Gwichʼin -gwj I L /Gwi -gwm I E Awngthim -gwn I L Gwandara -gwr I L Gwere -gwt I L Gawar-Bati -gwu I E Guwamu -gww I L Kwini -gwx I L Gua -gxx I L Wè Southern -gya I L Northwest Gbaya -gyb I L Garus -gyd I L Kayardild -gye I L Gyem -gyf I E Gungabula -gyg I L Gbayi -gyi I L Gyele -gyl I L Gayil -gym I L Ngäbere -gyn I L Guyanese Creole English -gyr I L Guarayu -gyy I E Gunya -gza I L Ganza -gzi I L Gazi -gzn I L Gane -haa I L Han -hab I L Hanoi Sign Language -hac I L Gurani -had I L Hatam -hae I L Eastern Oromo -haf I L Haiphong Sign Language -hag I L Hanga -hah I L Hahon -hai hai hai M L Haida -haj I L Hajong -hak I L Hakka Chinese -hal I L Halang -ham I L Hewa -han I L Hangaza -hao I L Hakö -hap I L Hupla -haq I L Ha -har I L Harari -has I L Haisla -hat hat hat ht I L Haitian -hau hau hau ha I L Hausa -hav I L Havu -haw haw haw I L Hawaiian -hax I L Southern Haida -hay I L Haya -haz I L Hazaragi -hba I L Hamba -hbb I L Huba -hbn I L Heiban -hbo I H Ancient Hebrew -hbs sh M L Serbo-Croatian Code element for 639-1 has been deprecated -hbu I L Habu -hca I L Andaman Creole Hindi -hch I L Huichol -hdn I L Northern Haida -hds I L Honduras Sign Language -hdy I L Hadiyya -hea I L Northern Qiandong Miao -heb heb heb he I L Hebrew -hed I L Herdé -heg I L Helong -heh I L Hehe -hei I L Heiltsuk -hem I L Hemba -her her her hz I L Herero -hgm I L Hai//om -hgw I L Haigwai -hhi I L Hoia Hoia -hhr I L Kerak -hhy I L Hoyahoya -hia I L Lamang -hib I E Hibito -hid I L Hidatsa -hif I L Fiji Hindi -hig I L Kamwe -hih I L Pamosu -hii I L Hinduri -hij I L Hijuk -hik I L Seit-Kaitetu -hil hil hil I L Hiligaynon -hin hin hin hi I L Hindi -hio I L Tsoa -hir I L Himarimã -hit hit hit I A Hittite -hiw I L Hiw -hix I L Hixkaryána -hji I L Haji -hka I L Kahe -hke I L Hunde -hkk I L Hunjara-Kaina Ke -hks I L Hong Kong Sign Language -hla I L Halia -hlb I L Halbi -hld I L Halang Doan -hle I L Hlersu -hlt I L Matu Chin -hlu I A Hieroglyphic Luwian -hma I L Southern Mashan Hmong -hmb I L Humburi Senni Songhay -hmc I L Central Huishui Hmong -hmd I L Large Flowery Miao -hme I L Eastern Huishui Hmong -hmf I L Hmong Don -hmg I L Southwestern Guiyang Hmong -hmh I L Southwestern Huishui Hmong -hmi I L Northern Huishui Hmong -hmj I L Ge -hmk I E Maek -hml I L Luopohe Hmong -hmm I L Central Mashan Hmong -hmn hmn hmn M L Hmong -hmo hmo hmo ho I L Hiri Motu -hmp I L Northern Mashan Hmong -hmq I L Eastern Qiandong Miao -hmr I L Hmar -hms I L Southern Qiandong Miao -hmt I L Hamtai -hmu I L Hamap -hmv I L Hmong Dô -hmw I L Western Mashan Hmong -hmy I L Southern Guiyang Hmong -hmz I L Hmong Shua -hna I L Mina (Cameroon) -hnd I L Southern Hindko -hne I L Chhattisgarhi -hnh I L //Ani -hni I L Hani -hnj I L Hmong Njua -hnn I L Hanunoo -hno I L Northern Hindko -hns I L Caribbean Hindustani -hnu I L Hung -hoa I L Hoava -hob I L Mari (Madang Province) -hoc I L Ho -hod I E Holma -hoe I L Horom -hoh I L Hobyót -hoi I L Holikachuk -hoj I L Hadothi -hol I L Holu -hom I E Homa -hoo I L Holoholo -hop I L Hopi -hor I E Horo -hos I L Ho Chi Minh City Sign Language -hot I L Hote -hov I L Hovongan -how I L Honi -hoy I L Holiya -hoz I L Hozo -hpo I L Hpon -hps I L Hawai'i Pidgin Sign Language -hra I L Hrangkhol -hrc I L Niwer Mil -hre I L Hre -hrk I L Haruku -hrm I L Horned Miao -hro I L Haroi -hrp I E Nhirrpi -hrt I L Hértevin -hru I L Hruso -hrv hrv hrv hr I L Croatian -hrw I L Warwar Feni -hrx I L Hunsrik -hrz I L Harzani -hsb hsb hsb I L Upper Sorbian -hsh I L Hungarian Sign Language -hsl I L Hausa Sign Language -hsn I L Xiang Chinese -hss I L Harsusi -hti I L Hoti -hto I L Minica Huitoto -hts I L Hadza -htu I L Hitu -htx I A Middle Hittite -hub I L Huambisa -huc I L =/Hua -hud I L Huaulu -hue I L San Francisco Del Mar Huave -huf I L Humene -hug I L Huachipaeri -huh I L Huilliche -hui I L Huli -huj I L Northern Guiyang Hmong -huk I L Hulung -hul I L Hula -hum I L Hungana -hun hun hun hu I L Hungarian -huo I L Hu -hup hup hup I L Hupa -huq I L Tsat -hur I L Halkomelem -hus I L Huastec -hut I L Humla -huu I L Murui Huitoto -huv I L San Mateo Del Mar Huave -huw I E Hukumina -hux I L Nüpode Huitoto -huy I L Hulaulá -huz I L Hunzib -hvc I L Haitian Vodoun Culture Language -hve I L San Dionisio Del Mar Huave -hvk I L Haveke -hvn I L Sabu -hvv I L Santa María Del Mar Huave -hwa I L Wané -hwc I L Hawai'i Creole English -hwo I L Hwana -hya I L Hya -hye arm hye hy I L Armenian -iai I L Iaai -ian I L Iatmul -iap I L Iapama -iar I L Purari -iba iba iba I L Iban -ibb I L Ibibio -ibd I L Iwaidja -ibe I L Akpes -ibg I L Ibanag -ibl I L Ibaloi -ibm I L Agoi -ibn I L Ibino -ibo ibo ibo ig I L Igbo -ibr I L Ibuoro -ibu I L Ibu -iby I L Ibani -ica I L Ede Ica -ich I L Etkywan -icl I L Icelandic Sign Language -icr I L Islander Creole English -ida I L Idakho-Isukha-Tiriki -idb I L Indo-Portuguese -idc I L Idon -idd I L Ede Idaca -ide I L Idere -idi I L Idi -ido ido ido io I C Ido -idr I L Indri -ids I L Idesa -idt I L Idaté -idu I L Idoma -ifa I L Amganad Ifugao -ifb I L Batad Ifugao -ife I L Ifè -iff I E Ifo -ifk I L Tuwali Ifugao -ifm I L Teke-Fuumu -ifu I L Mayoyao Ifugao -ify I L Keley-I Kallahan -igb I L Ebira -ige I L Igede -igg I L Igana -igl I L Igala -igm I L Kanggape -ign I L Ignaciano -igo I L Isebe -igs I C Interglossa -igw I L Igwe -ihb I L Iha Based Pidgin -ihi I L Ihievbe -ihp I L Iha -ihw I E Bidhawal -iii iii iii ii I L Sichuan Yi -iin I E Thiin -ijc I L Izon -ije I L Biseni -ijj I L Ede Ije -ijn I L Kalabari -ijs I L Southeast Ijo -ike I L Eastern Canadian Inuktitut -iki I L Iko -ikk I L Ika -ikl I L Ikulu -iko I L Olulumo-Ikom -ikp I L Ikpeshi -ikr I E Ikaranggal -ikt I L Inuinnaqtun -iku iku iku iu M L Inuktitut -ikv I L Iku-Gora-Ankwa -ikw I L Ikwere -ikx I L Ik -ikz I L Ikizu -ila I L Ile Ape -ilb I L Ila -ile ile ile ie I C Interlingue -ilg I E Garig-Ilgar -ili I L Ili Turki -ilk I L Ilongot -ill I L Iranun -ilo ilo ilo I L Iloko -ils I L International Sign -ilu I L Ili'uun -ilv I L Ilue -ima I L Mala Malasar -ime I L Imeraguen -imi I L Anamgura -iml I E Miluk -imn I L Imonda -imo I L Imbongu -imr I L Imroing -ims I A Marsian -imy I A Milyan -ina ina ina ia I C Interlingua (International Auxiliary Language Association) -inb I L Inga -ind ind ind id I L Indonesian -ing I L Degexit'an -inh inh inh I L Ingush -inj I L Jungle Inga -inl I L Indonesian Sign Language -inm I A Minaean -inn I L Isinai -ino I L Inoke-Yate -inp I L Iñapari -ins I L Indian Sign Language -int I L Intha -inz I E Ineseño -ior I L Inor -iou I L Tuma-Irumu -iow I E Iowa-Oto -ipi I L Ipili -ipk ipk ipk ik M L Inupiaq -ipo I L Ipiko -iqu I L Iquito -iqw I L Ikwo -ire I L Iresim -irh I L Irarutu -iri I L Irigwe -irk I L Iraqw -irn I L Irántxe -irr I L Ir -iru I L Irula -irx I L Kamberau -iry I L Iraya -isa I L Isabi -isc I L Isconahua -isd I L Isnag -ise I L Italian Sign Language -isg I L Irish Sign Language -ish I L Esan -isi I L Nkem-Nkum -isk I L Ishkashimi -isl ice isl is I L Icelandic -ism I L Masimasi -isn I L Isanzu -iso I L Isoko -isr I L Israeli Sign Language -ist I L Istriot -isu I L Isu (Menchum Division) -ita ita ita it I L Italian -itb I L Binongan Itneg -ite I E Itene -iti I L Inlaod Itneg -itk I L Judeo-Italian -itl I L Itelmen -itm I L Itu Mbon Uzo -ito I L Itonama -itr I L Iteri -its I L Isekiri -itt I L Maeng Itneg -itv I L Itawit -itw I L Ito -itx I L Itik -ity I L Moyadan Itneg -itz I L Itzá -ium I L Iu Mien -ivb I L Ibatan -ivv I L Ivatan -iwk I L I-Wak -iwm I L Iwam -iwo I L Iwur -iws I L Sepik Iwam -ixc I L Ixcatec -ixl I L Ixil -iya I L Iyayu -iyo I L Mesaka -iyx I L Yaka (Congo) -izh I L Ingrian -izr I L Izere -izz I L Izii -jaa I L Jamamadí -jab I L Hyam -jac I L Popti' -jad I L Jahanka -jae I L Yabem -jaf I L Jara -jah I L Jah Hut -jaj I L Zazao -jak I L Jakun -jal I L Yalahatan -jam I L Jamaican Creole English -jan I E Jandai -jao I L Yanyuwa -jaq I L Yaqay -jas I L New Caledonian Javanese -jat I L Jakati -jau I L Yaur -jav jav jav jv I L Javanese -jax I L Jambi Malay -jay I L Yan-nhangu -jaz I L Jawe -jbe I L Judeo-Berber -jbi I E Badjiri -jbj I L Arandai -jbk I L Barikewa -jbn I L Nafusi -jbo jbo jbo I C Lojban -jbr I L Jofotek-Bromnya -jbt I L Jabutí -jbu I L Jukun Takum -jbw I E Yawijibaya -jcs I L Jamaican Country Sign Language -jct I L Krymchak -jda I L Jad -jdg I L Jadgali -jdt I L Judeo-Tat -jeb I L Jebero -jee I L Jerung -jeg I L Jeng -jeh I L Jeh -jei I L Yei -jek I L Jeri Kuo -jel I L Yelmek -jen I L Dza -jer I L Jere -jet I L Manem -jeu I L Jonkor Bourmataguil -jgb I E Ngbee -jge I L Judeo-Georgian -jgk I L Gwak -jgo I L Ngomba -jhi I L Jehai -jhs I L Jhankot Sign Language -jia I L Jina -jib I L Jibu -jic I L Tol -jid I L Bu -jie I L Jilbe -jig I L Djingili -jih I L sTodsde -jii I L Jiiddu -jil I L Jilim -jim I L Jimi (Cameroon) -jio I L Jiamao -jiq I L Guanyinqiao -jit I L Jita -jiu I L Youle Jinuo -jiv I L Shuar -jiy I L Buyuan Jinuo -jjr I L Bankal -jkm I L Mobwa Karen -jko I L Kubo -jkp I L Paku Karen -jkr I L Koro (India) -jku I L Labir -jle I L Ngile -jls I L Jamaican Sign Language -jma I L Dima -jmb I L Zumbun -jmc I L Machame -jmd I L Yamdena -jmi I L Jimi (Nigeria) -jml I L Jumli -jmn I L Makuri Naga -jmr I L Kamara -jms I L Mashi (Nigeria) -jmw I L Mouwase -jmx I L Western Juxtlahuaca Mixtec -jna I L Jangshung -jnd I L Jandavra -jng I E Yangman -jni I L Janji -jnj I L Yemsa -jnl I L Rawat -jns I L Jaunsari -job I L Joba -jod I L Wojenaka -jor I E Jorá -jos I L Jordanian Sign Language -jow I L Jowulu -jpa I H Jewish Palestinian Aramaic -jpn jpn jpn ja I L Japanese -jpr jpr jpr I L Judeo-Persian -jqr I L Jaqaru -jra I L Jarai -jrb jrb jrb M L Judeo-Arabic -jrr I L Jiru -jrt I L Jorto -jru I L Japrería -jsl I L Japanese Sign Language -jua I L Júma -jub I L Wannu -juc I E Jurchen -jud I L Worodougou -juh I L Hõne -jui I E Ngadjuri -juk I L Wapan -jul I L Jirel -jum I L Jumjum -jun I L Juang -juo I L Jiba -jup I L Hupdë -jur I L Jurúna -jus I L Jumla Sign Language -jut I L Jutish -juu I L Ju -juw I L Wãpha -juy I L Juray -jvd I E Javindo -jvn I L Caribbean Javanese -jwi I L Jwira-Pepesa -jya I L Jiarong -jye I L Judeo-Yemeni Arabic -jyy I L Jaya -kaa kaa kaa I L Kara-Kalpak -kab kab kab I L Kabyle -kac kac kac I L Kachin -kad I L Adara -kae I E Ketangalan -kaf I L Katso -kag I L Kajaman -kah I L Kara (Central African Republic) -kai I L Karekare -kaj I L Jju -kak I L Kayapa Kallahan -kal kal kal kl I L Kalaallisut -kam kam kam I L Kamba (Kenya) -kan kan kan kn I L Kannada -kao I L Xaasongaxango -kap I L Bezhta -kaq I L Capanahua -kas kas kas ks I L Kashmiri -kat geo kat ka I L Georgian -kau kau kau kr M L Kanuri -kav I L Katukína -kaw kaw kaw I A Kawi -kax I L Kao -kay I L Kamayurá -kaz kaz kaz kk I L Kazakh -kba I E Kalarko -kbb I E Kaxuiâna -kbc I L Kadiwéu -kbd kbd kbd I L Kabardian -kbe I L Kanju -kbf I E Kakauhua -kbg I L Khamba -kbh I L Camsá -kbi I L Kaptiau -kbj I L Kari -kbk I L Grass Koiari -kbl I L Kanembu -kbm I L Iwal -kbn I L Kare (Central African Republic) -kbo I L Keliko -kbp I L Kabiyè -kbq I L Kamano -kbr I L Kafa -kbs I L Kande -kbt I L Abadi -kbu I L Kabutra -kbv I L Dera (Indonesia) -kbw I L Kaiep -kbx I L Ap Ma -kby I L Manga Kanuri -kbz I L Duhwa -kca I L Khanty -kcb I L Kawacha -kcc I L Lubila -kcd I L Ngkâlmpw Kanum -kce I L Kaivi -kcf I L Ukaan -kcg I L Tyap -kch I L Vono -kci I L Kamantan -kcj I L Kobiana -kck I L Kalanga -kcl I L Kela (Papua New Guinea) -kcm I L Gula (Central African Republic) -kcn I L Nubi -kco I L Kinalakna -kcp I L Kanga -kcq I L Kamo -kcr I L Katla -kcs I L Koenoem -kct I L Kaian -kcu I L Kami (Tanzania) -kcv I L Kete -kcw I L Kabwari -kcx I L Kachama-Ganjule -kcy I L Korandje -kcz I L Konongo -kda I E Worimi -kdc I L Kutu -kdd I L Yankunytjatjara -kde I L Makonde -kdf I L Mamusi -kdg I L Seba -kdh I L Tem -kdi I L Kumam -kdj I L Karamojong -kdk I L Numèè -kdl I L Tsikimba -kdm I L Kagoma -kdn I L Kunda -kdp I L Kaningdon-Nindem -kdq I L Koch -kdr I L Karaim -kdt I L Kuy -kdu I L Kadaru -kdw I L Koneraw -kdx I L Kam -kdy I L Keder -kdz I L Kwaja -kea I L Kabuverdianu -keb I L Kélé -kec I L Keiga -ked I L Kerewe -kee I L Eastern Keres -kef I L Kpessi -keg I L Tese -keh I L Keak -kei I L Kei -kej I L Kadar -kek I L Kekchí -kel I L Kela (Democratic Republic of Congo) -kem I L Kemak -ken I L Kenyang -keo I L Kakwa -kep I L Kaikadi -keq I L Kamar -ker I L Kera -kes I L Kugbo -ket I L Ket -keu I L Akebu -kev I L Kanikkaran -kew I L West Kewa -kex I L Kukna -key I L Kupia -kez I L Kukele -kfa I L Kodava -kfb I L Northwestern Kolami -kfc I L Konda-Dora -kfd I L Korra Koraga -kfe I L Kota (India) -kff I L Koya -kfg I L Kudiya -kfh I L Kurichiya -kfi I L Kannada Kurumba -kfj I L Kemiehua -kfk I L Kinnauri -kfl I L Kung -kfm I L Khunsari -kfn I L Kuk -kfo I L Koro (Côte d'Ivoire) -kfp I L Korwa -kfq I L Korku -kfr I L Kachchi -kfs I L Bilaspuri -kft I L Kanjari -kfu I L Katkari -kfv I L Kurmukar -kfw I L Kharam Naga -kfx I L Kullu Pahari -kfy I L Kumaoni -kfz I L Koromfé -kga I L Koyaga -kgb I L Kawe -kgc I L Kasseng -kgd I L Kataang -kge I L Komering -kgf I L Kube -kgg I L Kusunda -kgi I L Selangor Sign Language -kgj I L Gamale Kham -kgk I L Kaiwá -kgl I E Kunggari -kgm I E Karipúna -kgn I L Karingani -kgo I L Krongo -kgp I L Kaingang -kgq I L Kamoro -kgr I L Abun -kgs I L Kumbainggar -kgt I L Somyev -kgu I L Kobol -kgv I L Karas -kgw I L Karon Dori -kgx I L Kamaru -kgy I L Kyerung -kha kha kha I L Khasi -khb I L Lü -khc I L Tukang Besi North -khd I L Bädi Kanum -khe I L Korowai -khf I L Khuen -khg I L Khams Tibetan -khh I L Kehu -khj I L Kuturmi -khk I L Halh Mongolian -khl I L Lusi -khm khm khm km I L Central Khmer -khn I L Khandesi -kho kho kho I A Khotanese -khp I L Kapori -khq I L Koyra Chiini Songhay -khr I L Kharia -khs I L Kasua -kht I L Khamti -khu I L Nkhumbi -khv I L Khvarshi -khw I L Khowar -khx I L Kanu -khy I L Kele (Democratic Republic of Congo) -khz I L Keapara -kia I L Kim -kib I L Koalib -kic I L Kickapoo -kid I L Koshin -kie I L Kibet -kif I L Eastern Parbate Kham -kig I L Kimaama -kih I L Kilmeri -kii I E Kitsai -kij I L Kilivila -kik kik kik ki I L Kikuyu -kil I L Kariya -kim I L Karagas -kin kin kin rw I L Kinyarwanda -kio I L Kiowa -kip I L Sheshi Kham -kiq I L Kosadle -kir kir kir ky I L Kirghiz -kis I L Kis -kit I L Agob -kiu I L Kirmanjki (individual language) -kiv I L Kimbu -kiw I L Northeast Kiwai -kix I L Khiamniungan Naga -kiy I L Kirikiri -kiz I L Kisi -kja I L Mlap -kjb I L Q'anjob'al -kjc I L Coastal Konjo -kjd I L Southern Kiwai -kje I L Kisar -kjf I L Khalaj -kjg I L Khmu -kjh I L Khakas -kji I L Zabana -kjj I L Khinalugh -kjk I L Highland Konjo -kjl I L Western Parbate Kham -kjm I L Kháng -kjn I L Kunjen -kjo I L Harijan Kinnauri -kjp I L Pwo Eastern Karen -kjq I L Western Keres -kjr I L Kurudu -kjs I L East Kewa -kjt I L Phrae Pwo Karen -kju I L Kashaya -kjx I L Ramopa -kjy I L Erave -kjz I L Bumthangkha -kka I L Kakanda -kkb I L Kwerisa -kkc I L Odoodee -kkd I L Kinuku -kke I L Kakabe -kkf I L Kalaktang Monpa -kkg I L Mabaka Valley Kalinga -kkh I L Khün -kki I L Kagulu -kkj I L Kako -kkk I L Kokota -kkl I L Kosarek Yale -kkm I L Kiong -kkn I L Kon Keu -kko I L Karko -kkp I L Gugubera -kkq I L Kaiku -kkr I L Kir-Balar -kks I L Giiwo -kkt I L Koi -kku I L Tumi -kkv I L Kangean -kkw I L Teke-Kukuya -kkx I L Kohin -kky I L Guguyimidjir -kkz I L Kaska -kla I E Klamath-Modoc -klb I L Kiliwa -klc I L Kolbila -kld I L Gamilaraay -kle I L Kulung (Nepal) -klf I L Kendeje -klg I L Tagakaulo -klh I L Weliki -kli I L Kalumpang -klj I L Turkic Khalaj -klk I L Kono (Nigeria) -kll I L Kagan Kalagan -klm I L Migum -kln M L Kalenjin -klo I L Kapya -klp I L Kamasa -klq I L Rumu -klr I L Khaling -kls I L Kalasha -klt I L Nukna -klu I L Klao -klv I L Maskelynes -klw I L Lindu -klx I L Koluwawa -kly I L Kalao -klz I L Kabola -kma I L Konni -kmb kmb kmb I L Kimbundu -kmc I L Southern Dong -kmd I L Majukayang Kalinga -kme I L Bakole -kmf I L Kare (Papua New Guinea) -kmg I L Kâte -kmh I L Kalam -kmi I L Kami (Nigeria) -kmj I L Kumarbhag Paharia -kmk I L Limos Kalinga -kml I L Tanudan Kalinga -kmm I L Kom (India) -kmn I L Awtuw -kmo I L Kwoma -kmp I L Gimme -kmq I L Kwama -kmr I L Northern Kurdish -kms I L Kamasau -kmt I L Kemtuik -kmu I L Kanite -kmv I L Karipúna Creole French -kmw I L Komo (Democratic Republic of Congo) -kmx I L Waboda -kmy I L Koma -kmz I L Khorasani Turkish -kna I L Dera (Nigeria) -knb I L Lubuagan Kalinga -knc I L Central Kanuri -knd I L Konda -kne I L Kankanaey -knf I L Mankanya -kng I L Koongo -kni I L Kanufi -knj I L Western Kanjobal -knk I L Kuranko -knl I L Keninjal -knm I L Kanamarí -knn I L Konkani (individual language) -kno I L Kono (Sierra Leone) -knp I L Kwanja -knq I L Kintaq -knr I L Kaningra -kns I L Kensiu -knt I L Panoan Katukína -knu I L Kono (Guinea) -knv I L Tabo -knw I L Kung-Ekoka -knx I L Kendayan -kny I L Kanyok -knz I L Kalamsé -koa I L Konomala -koc I E Kpati -kod I L Kodi -koe I L Kacipo-Balesi -kof I E Kubi -kog I L Cogui -koh I L Koyo -koi I L Komi-Permyak -koj I L Sara Dunjo -kok kok kok M L Konkani (macrolanguage) -kol I L Kol (Papua New Guinea) -kom kom kom kv M L Komi -kon kon kon kg M L Kongo -koo I L Konzo -kop I L Waube -koq I L Kota (Gabon) -kor kor kor ko I L Korean -kos kos kos I L Kosraean -kot I L Lagwan -kou I L Koke -kov I L Kudu-Camo -kow I L Kugama -kox I E Coxima -koy I L Koyukon -koz I L Korak -kpa I L Kutto -kpb I L Mullu Kurumba -kpc I L Curripaco -kpd I L Koba -kpe kpe kpe M L Kpelle -kpf I L Komba -kpg I L Kapingamarangi -kph I L Kplang -kpi I L Kofei -kpj I L Karajá -kpk I L Kpan -kpl I L Kpala -kpm I L Koho -kpn I E Kepkiriwát -kpo I L Ikposo -kpq I L Korupun-Sela -kpr I L Korafe-Yegha -kps I L Tehit -kpt I L Karata -kpu I L Kafoa -kpv I L Komi-Zyrian -kpw I L Kobon -kpx I L Mountain Koiali -kpy I L Koryak -kpz I L Kupsabiny -kqa I L Mum -kqb I L Kovai -kqc I L Doromu-Koki -kqd I L Koy Sanjaq Surat -kqe I L Kalagan -kqf I L Kakabai -kqg I L Khe -kqh I L Kisankasa -kqi I L Koitabu -kqj I L Koromira -kqk I L Kotafon Gbe -kql I L Kyenele -kqm I L Khisa -kqn I L Kaonde -kqo I L Eastern Krahn -kqp I L Kimré -kqq I L Krenak -kqr I L Kimaragang -kqs I L Northern Kissi -kqt I L Klias River Kadazan -kqu I E Seroa -kqv I L Okolod -kqw I L Kandas -kqx I L Mser -kqy I L Koorete -kqz I E Korana -kra I L Kumhali -krb I E Karkin -krc krc krc I L Karachay-Balkar -krd I L Kairui-Midiki -kre I L Panará -krf I L Koro (Vanuatu) -krh I L Kurama -kri I L Krio -krj I L Kinaray-A -krk I E Kerek -krl krl krl I L Karelian -krm I L Krim -krn I L Sapo -krp I L Korop -krr I L Kru'ng 2 -krs I L Gbaya (Sudan) -krt I L Tumari Kanuri -kru kru kru I L Kurukh -krv I L Kavet -krw I L Western Krahn -krx I L Karon -kry I L Kryts -krz I L Sota Kanum -ksa I L Shuwa-Zamani -ksb I L Shambala -ksc I L Southern Kalinga -ksd I L Kuanua -kse I L Kuni -ksf I L Bafia -ksg I L Kusaghe -ksh I L Kölsch -ksi I L Krisa -ksj I L Uare -ksk I L Kansa -ksl I L Kumalu -ksm I L Kumba -ksn I L Kasiguranin -kso I L Kofa -ksp I L Kaba -ksq I L Kwaami -ksr I L Borong -kss I L Southern Kisi -kst I L Winyé -ksu I L Khamyang -ksv I L Kusu -ksw I L S'gaw Karen -ksx I L Kedang -ksy I L Kharia Thar -ksz I L Kodaku -kta I L Katua -ktb I L Kambaata -ktc I L Kholok -ktd I L Kokata -kte I L Nubri -ktf I L Kwami -ktg I E Kalkutung -kth I L Karanga -kti I L North Muyu -ktj I L Plapo Krumen -ktk I E Kaniet -ktl I L Koroshi -ktm I L Kurti -ktn I L Karitiâna -kto I L Kuot -ktp I L Kaduo -ktq I E Katabaga -ktr I L Kota Marudu Tinagas -kts I L South Muyu -ktt I L Ketum -ktu I L Kituba (Democratic Republic of Congo) -ktv I L Eastern Katu -ktw I E Kato -ktx I L Kaxararí -kty I L Kango (Bas-Uélé District) -ktz I L Ju/'hoan -kua kua kua kj I L Kuanyama -kub I L Kutep -kuc I L Kwinsu -kud I L 'Auhelawa -kue I L Kuman -kuf I L Western Katu -kug I L Kupa -kuh I L Kushi -kui I L Kuikúro-Kalapálo -kuj I L Kuria -kuk I L Kepo' -kul I L Kulere -kum kum kum I L Kumyk -kun I L Kunama -kuo I L Kumukio -kup I L Kunimaipa -kuq I L Karipuna -kur kur kur ku M L Kurdish -kus I L Kusaal -kut kut kut I L Kutenai -kuu I L Upper Kuskokwim -kuv I L Kur -kuw I L Kpagua -kux I L Kukatja -kuy I L Kuuku-Ya'u -kuz I E Kunza -kva I L Bagvalal -kvb I L Kubu -kvc I L Kove -kvd I L Kui (Indonesia) -kve I L Kalabakan -kvf I L Kabalai -kvg I L Kuni-Boazi -kvh I L Komodo -kvi I L Kwang -kvj I L Psikye -kvk I L Korean Sign Language -kvl I L Kayaw -kvm I L Kendem -kvn I L Border Kuna -kvo I L Dobel -kvp I L Kompane -kvq I L Geba Karen -kvr I L Kerinci -kvs I L Kunggara -kvt I L Lahta Karen -kvu I L Yinbaw Karen -kvv I L Kola -kvw I L Wersing -kvx I L Parkari Koli -kvy I L Yintale Karen -kvz I L Tsakwambo -kwa I L Dâw -kwb I L Kwa -kwc I L Likwala -kwd I L Kwaio -kwe I L Kwerba -kwf I L Kwara'ae -kwg I L Sara Kaba Deme -kwh I L Kowiai -kwi I L Awa-Cuaiquer -kwj I L Kwanga -kwk I L Kwakiutl -kwl I L Kofyar -kwm I L Kwambi -kwn I L Kwangali -kwo I L Kwomtari -kwp I L Kodia -kwq I L Kwak -kwr I L Kwer -kws I L Kwese -kwt I L Kwesten -kwu I L Kwakum -kwv I L Sara Kaba Náà -kww I L Kwinti -kwx I L Khirwar -kwy I L San Salvador Kongo -kwz I E Kwadi -kxa I L Kairiru -kxb I L Krobu -kxc I L Konso -kxd I L Brunei -kxe I L Kakihum -kxf I L Manumanaw Karen -kxh I L Karo (Ethiopia) -kxi I L Keningau Murut -kxj I L Kulfa -kxk I L Zayein Karen -kxl I L Nepali Kurux -kxm I L Northern Khmer -kxn I L Kanowit-Tanjong Melanau -kxo I E Kanoé -kxp I L Wadiyara Koli -kxq I L Smärky Kanum -kxr I L Koro (Papua New Guinea) -kxs I L Kangjia -kxt I L Koiwat -kxu I L Kui (India) -kxv I L Kuvi -kxw I L Konai -kxx I L Likuba -kxy I L Kayong -kxz I L Kerewo -kya I L Kwaya -kyb I L Butbut Kalinga -kyc I L Kyaka -kyd I L Karey -kye I L Krache -kyf I L Kouya -kyg I L Keyagana -kyh I L Karok -kyi I L Kiput -kyj I L Karao -kyk I L Kamayo -kyl I L Kalapuya -kym I L Kpatili -kyn I L Northern Binukidnon -kyo I L Kelon -kyp I L Kang -kyq I L Kenga -kyr I L Kuruáya -kys I L Baram Kayan -kyt I L Kayagar -kyu I L Western Kayah -kyv I L Kayort -kyw I L Kudmali -kyx I L Rapoisi -kyy I L Kambaira -kyz I L Kayabí -kza I L Western Karaboro -kzb I L Kaibobo -kzc I L Bondoukou Kulango -kzd I L Kadai -kze I L Kosena -kzf I L Da'a Kaili -kzg I L Kikai -kzi I L Kelabit -kzj I L Coastal Kadazan -kzk I E Kazukuru -kzl I L Kayeli -kzm I L Kais -kzn I L Kokola -kzo I L Kaningi -kzp I L Kaidipang -kzq I L Kaike -kzr I L Karang -kzs I L Sugut Dusun -kzt I L Tambunan Dusun -kzu I L Kayupulau -kzv I L Komyandaret -kzw I E Karirí-Xocó -kzx I L Kamarian -kzy I L Kango (Tshopo District) -kzz I L Kalabra -laa I L Southern Subanen -lab I A Linear A -lac I L Lacandon -lad lad lad I L Ladino -lae I L Pattani -laf I L Lafofa -lag I L Langi -lah lah lah M L Lahnda -lai I L Lambya -laj I L Lango (Uganda) -lak I L Laka (Nigeria) -lal I L Lalia -lam lam lam I L Lamba -lan I L Laru -lao lao lao lo I L Lao -lap I L Laka (Chad) -laq I L Qabiao -lar I L Larteh -las I L Lama (Togo) -lat lat lat la I A Latin -lau I L Laba -lav lav lav lv M L Latvian -law I L Lauje -lax I L Tiwa -lay I L Lama (Myanmar) -laz I E Aribwatsa -lba I E Lui -lbb I L Label -lbc I L Lakkia -lbe I L Lak -lbf I L Tinani -lbg I L Laopang -lbi I L La'bi -lbj I L Ladakhi -lbk I L Central Bontok -lbl I L Libon Bikol -lbm I L Lodhi -lbn I L Lamet -lbo I L Laven -lbq I L Wampar -lbr I L Lohorung -lbs I L Libyan Sign Language -lbt I L Lachi -lbu I L Labu -lbv I L Lavatbura-Lamusong -lbw I L Tolaki -lbx I L Lawangan -lby I E Lamu-Lamu -lbz I L Lardil -lcc I L Legenyem -lcd I L Lola -lce I L Loncong -lcf I L Lubu -lch I L Luchazi -lcl I L Lisela -lcm I L Tungag -lcp I L Western Lawa -lcq I L Luhu -lcs I L Lisabata-Nuniali -lda I L Kla-Dan -ldb I L Dũya -ldd I L Luri -ldg I L Lenyima -ldh I L Lamja-Dengsa-Tola -ldi I L Laari -ldj I L Lemoro -ldk I L Leelau -ldl I L Kaan -ldm I L Landoma -ldn I C Láadan -ldo I L Loo -ldp I L Tso -ldq I L Lufu -lea I L Lega-Shabunda -leb I L Lala-Bisa -lec I L Leco -led I L Lendu -lee I L Lyélé -lef I L Lelemi -leg I L Lengua -leh I L Lenje -lei I L Lemio -lej I L Lengola -lek I L Leipon -lel I L Lele (Democratic Republic of Congo) -lem I L Nomaande -len I E Lenca -leo I L Leti (Cameroon) -lep I L Lepcha -leq I L Lembena -ler I L Lenkau -les I L Lese -let I L Lesing-Gelimi -leu I L Kara (Papua New Guinea) -lev I L Lamma -lew I L Ledo Kaili -lex I L Luang -ley I L Lemolang -lez lez lez I L Lezghian -lfa I L Lefa -lfn I C Lingua Franca Nova -lga I L Lungga -lgb I L Laghu -lgg I L Lugbara -lgh I L Laghuu -lgi I L Lengilu -lgk I L Lingarak -lgl I L Wala -lgm I L Lega-Mwenga -lgn I L Opuuo -lgq I L Logba -lgr I L Lengo -lgt I L Pahi -lgu I L Longgu -lgz I L Ligenza -lha I L Laha (Viet Nam) -lhh I L Laha (Indonesia) -lhi I L Lahu Shi -lhl I L Lahul Lohar -lhm I L Lhomi -lhn I L Lahanan -lhp I L Lhokpu -lhs I E Mlahsö -lht I L Lo-Toga -lhu I L Lahu -lia I L West-Central Limba -lib I L Likum -lic I L Hlai -lid I L Nyindrou -lie I L Likila -lif I L Limbu -lig I L Ligbi -lih I L Lihir -lii I L Lingkhim -lij I L Ligurian -lik I L Lika -lil I L Lillooet -lim lim lim li I L Limburgan -lin lin lin ln I L Lingala -lio I L Liki -lip I L Sekpele -liq I L Libido -lir I L Liberian English -lis I L Lisu -lit lit lit lt I L Lithuanian -liu I L Logorik -liv I L Liv -liw I L Col -lix I L Liabuku -liy I L Banda-Bambari -liz I L Libinza -lja I E Golpa -lje I L Rampi -lji I L Laiyolo -ljl I L Li'o -ljp I L Lampung Api -ljw I L Yirandali -ljx I E Yuru -lka I L Lakalei -lkb I L Kabras -lkc I L Kucong -lkd I L Lakondê -lke I L Kenyi -lkh I L Lakha -lki I L Laki -lkj I L Remun -lkl I L Laeko-Libuat -lkm I E Kalaamaya -lkn I L Lakon -lko I L Khayo -lkr I L Päri -lks I L Kisa -lkt I L Lakota -lku I E Kungkari -lky I L Lokoya -lla I L Lala-Roba -llb I L Lolo -llc I L Lele (Guinea) -lld I L Ladin -lle I L Lele (Papua New Guinea) -llf I E Hermit -llg I L Lole -llh I L Lamu -lli I L Teke-Laali -llj I E Ladji Ladji -llk I E Lelak -lll I L Lilau -llm I L Lasalimu -lln I L Lele (Chad) -llo I L Khlor -llp I L North Efate -llq I L Lolak -lls I L Lithuanian Sign Language -llu I L Lau -llx I L Lauan -lma I L East Limba -lmb I L Merei -lmc I E Limilngan -lmd I L Lumun -lme I L Pévé -lmf I L South Lembata -lmg I L Lamogai -lmh I L Lambichhong -lmi I L Lombi -lmj I L West Lembata -lmk I L Lamkang -lml I L Hano -lmm I L Lamam -lmn I L Lambadi -lmo I L Lombard -lmp I L Limbum -lmq I L Lamatuka -lmr I L Lamalera -lmu I L Lamenu -lmv I L Lomaiviti -lmw I L Lake Miwok -lmx I L Laimbue -lmy I L Lamboya -lmz I E Lumbee -lna I L Langbashe -lnb I L Mbalanhu -lnd I L Lundayeh -lng I A Langobardic -lnh I L Lanoh -lni I L Daantanai' -lnj I E Leningitij -lnl I L South Central Banda -lnm I L Langam -lnn I L Lorediakarkar -lno I L Lango (Sudan) -lns I L Lamnso' -lnu I L Longuda -lnw I E Lanima -lnz I L Lonzo -loa I L Loloda -lob I L Lobi -loc I L Inonhan -loe I L Saluan -lof I L Logol -log I L Logo -loh I L Narim -loi I L Loma (Côte d'Ivoire) -loj I L Lou -lok I L Loko -lol lol lol I L Mongo -lom I L Loma (Liberia) -lon I L Malawi Lomwe -loo I L Lombo -lop I L Lopa -loq I L Lobala -lor I L Téén -los I L Loniu -lot I L Otuho -lou I L Louisiana Creole French -lov I L Lopi -low I L Tampias Lobu -lox I L Loun -loy I L Loke -loz loz loz I L Lozi -lpa I L Lelepa -lpe I L Lepki -lpn I L Long Phuri Naga -lpo I L Lipo -lpx I L Lopit -lra I L Rara Bakati' -lrc I L Northern Luri -lre I E Laurentian -lrg I E Laragia -lri I L Marachi -lrk I L Loarki -lrl I L Lari -lrm I L Marama -lrn I L Lorang -lro I L Laro -lrr I L Southern Yamphu -lrt I L Larantuka Malay -lrv I L Larevat -lrz I L Lemerig -lsa I L Lasgerdi -lsd I L Lishana Deni -lse I L Lusengo -lsg I L Lyons Sign Language -lsh I L Lish -lsi I L Lashi -lsl I L Latvian Sign Language -lsm I L Saamia -lso I L Laos Sign Language -lsp I L Panamanian Sign Language -lsr I L Aruop -lss I L Lasi -lst I L Trinidad and Tobago Sign Language -lsy I L Mauritian Sign Language -ltc I H Late Middle Chinese -ltg I L Latgalian -lti I L Leti (Indonesia) -ltn I L Latundê -lto I L Tsotso -lts I L Tachoni -ltu I L Latu -ltz ltz ltz lb I L Luxembourgish -lua lua lua I L Luba-Lulua -lub lub lub lu I L Luba-Katanga -luc I L Aringa -lud I L Ludian -lue I L Luvale -luf I L Laua -lug lug lug lg I L Ganda -lui lui lui I L Luiseno -luj I L Luna -luk I L Lunanakha -lul I L Olu'bo -lum I L Luimbi -lun lun lun I L Lunda -luo luo luo I L Luo (Kenya and Tanzania) -lup I L Lumbu -luq I L Lucumi -lur I L Laura -lus lus lus I L Lushai -lut I L Lushootseed -luu I L Lumba-Yakkha -luv I L Luwati -luw I L Luo (Cameroon) -luy M L Luyia -luz I L Southern Luri -lva I L Maku'a -lvk I L Lavukaleve -lvs I L Standard Latvian -lvu I L Levuka -lwa I L Lwalu -lwe I L Lewo Eleng -lwg I L Wanga -lwh I L White Lachi -lwl I L Eastern Lawa -lwm I L Laomian -lwo I L Luwo -lwt I L Lewotobi -lwu I L Lawu -lww I L Lewo -lya I L Layakha -lyg I L Lyngngam -lyn I L Luyana -lzh I H Literary Chinese -lzl I L Litzlitz -lzn I L Leinong Naga -lzz I L Laz -maa I L San Jerónimo Tecóatl Mazatec -mab I L Yutanduchi Mixtec -mad mad mad I L Madurese -mae I L Bo-Rukul -maf I L Mafa -mag mag mag I L Magahi -mah mah mah mh I L Marshallese -mai mai mai I L Maithili -maj I L Jalapa De Díaz Mazatec -mak mak mak I L Makasar -mal mal mal ml I L Malayalam -mam I L Mam -man man man M L Mandingo -maq I L Chiquihuitlán Mazatec -mar mar mar mr I L Marathi -mas mas mas I L Masai -mat I L San Francisco Matlatzinca -mau I L Huautla Mazatec -mav I L Sateré-Mawé -maw I L Mampruli -max I L North Moluccan Malay -maz I L Central Mazahua -mba I L Higaonon -mbb I L Western Bukidnon Manobo -mbc I L Macushi -mbd I L Dibabawon Manobo -mbe I E Molale -mbf I L Baba Malay -mbh I L Mangseng -mbi I L Ilianen Manobo -mbj I L Nadëb -mbk I L Malol -mbl I L Maxakalí -mbm I L Ombamba -mbn I L Macaguán -mbo I L Mbo (Cameroon) -mbp I L Malayo -mbq I L Maisin -mbr I L Nukak Makú -mbs I L Sarangani Manobo -mbt I L Matigsalug Manobo -mbu I L Mbula-Bwazza -mbv I L Mbulungish -mbw I L Maring -mbx I L Mari (East Sepik Province) -mby I L Memoni -mbz I L Amoltepec Mixtec -mca I L Maca -mcb I L Machiguenga -mcc I L Bitur -mcd I L Sharanahua -mce I L Itundujia Mixtec -mcf I L Matsés -mcg I L Mapoyo -mch I L Maquiritari -mci I L Mese -mcj I L Mvanip -mck I L Mbunda -mcl I E Macaguaje -mcm I L Malaccan Creole Portuguese -mcn I L Masana -mco I L Coatlán Mixe -mcp I L Makaa -mcq I L Ese -mcr I L Menya -mcs I L Mambai -mct I L Mengisa -mcu I L Cameroon Mambila -mcv I L Minanibai -mcw I L Mawa (Chad) -mcx I L Mpiemo -mcy I L South Watut -mcz I L Mawan -mda I L Mada (Nigeria) -mdb I L Morigi -mdc I L Male (Papua New Guinea) -mdd I L Mbum -mde I L Maba (Chad) -mdf mdf mdf I L Moksha -mdg I L Massalat -mdh I L Maguindanaon -mdi I L Mamvu -mdj I L Mangbetu -mdk I L Mangbutu -mdl I L Maltese Sign Language -mdm I L Mayogo -mdn I L Mbati -mdp I L Mbala -mdq I L Mbole -mdr mdr mdr I L Mandar -mds I L Maria (Papua New Guinea) -mdt I L Mbere -mdu I L Mboko -mdv I L Santa Lucía Monteverde Mixtec -mdw I L Mbosi -mdx I L Dizin -mdy I L Male (Ethiopia) -mdz I L Suruí Do Pará -mea I L Menka -meb I L Ikobi -mec I L Mara -med I L Melpa -mee I L Mengen -mef I L Megam -meh I L Southwestern Tlaxiaco Mixtec -mei I L Midob -mej I L Meyah -mek I L Mekeo -mel I L Central Melanau -mem I E Mangala -men men men I L Mende (Sierra Leone) -meo I L Kedah Malay -mep I L Miriwung -meq I L Merey -mer I L Meru -mes I L Masmaje -met I L Mato -meu I L Motu -mev I L Mano -mew I L Maaka -mey I L Hassaniyya -mez I L Menominee -mfa I L Pattani Malay -mfb I L Bangka -mfc I L Mba -mfd I L Mendankwe-Nkwen -mfe I L Morisyen -mff I L Naki -mfg I L Mogofin -mfh I L Matal -mfi I L Wandala -mfj I L Mefele -mfk I L North Mofu -mfl I L Putai -mfm I L Marghi South -mfn I L Cross River Mbembe -mfo I L Mbe -mfp I L Makassar Malay -mfq I L Moba -mfr I L Marithiel -mfs I L Mexican Sign Language -mft I L Mokerang -mfu I L Mbwela -mfv I L Mandjak -mfw I E Mulaha -mfx I L Melo -mfy I L Mayo -mfz I L Mabaan -mga mga mga I H Middle Irish (900-1200) -mgb I L Mararit -mgc I L Morokodo -mgd I L Moru -mge I L Mango -mgf I L Maklew -mgg I L Mpumpong -mgh I L Makhuwa-Meetto -mgi I L Lijili -mgj I L Abureni -mgk I L Mawes -mgl I L Maleu-Kilenge -mgm I L Mambae -mgn I L Mbangi -mgo I L Meta' -mgp I L Eastern Magar -mgq I L Malila -mgr I L Mambwe-Lungu -mgs I L Manda (Tanzania) -mgt I L Mongol -mgu I L Mailu -mgv I L Matengo -mgw I L Matumbi -mgy I L Mbunga -mgz I L Mbugwe -mha I L Manda (India) -mhb I L Mahongwe -mhc I L Mocho -mhd I L Mbugu -mhe I L Besisi -mhf I L Mamaa -mhg I L Margu -mhh I L Maskoy Pidgin -mhi I L Ma'di -mhj I L Mogholi -mhk I L Mungaka -mhl I L Mauwake -mhm I L Makhuwa-Moniga -mhn I L Mócheno -mho I L Mashi (Zambia) -mhp I L Balinese Malay -mhq I L Mandan -mhr I L Eastern Mari -mhs I L Buru (Indonesia) -mht I L Mandahuaca -mhu I L Digaro-Mishmi -mhw I L Mbukushu -mhx I L Maru -mhy I L Ma'anyan -mhz I L Mor (Mor Islands) -mia I L Miami -mib I L Atatláhuca Mixtec -mic mic mic I L Mi'kmaq -mid I L Mandaic -mie I L Ocotepec Mixtec -mif I L Mofu-Gudur -mig I L San Miguel El Grande Mixtec -mih I L Chayuco Mixtec -mii I L Chigmecatitlán Mixtec -mij I L Abar -mik I L Mikasuki -mil I L Peñoles Mixtec -mim I L Alacatlatzala Mixtec -min min min I L Minangkabau -mio I L Pinotepa Nacional Mixtec -mip I L Apasco-Apoala Mixtec -miq I L Mískito -mir I L Isthmus Mixe -mis mis mis S S Uncoded languages -mit I L Southern Puebla Mixtec -miu I L Cacaloxtepec Mixtec -miw I L Akoye -mix I L Mixtepec Mixtec -miy I L Ayutla Mixtec -miz I L Coatzospan Mixtec -mjc I L San Juan Colorado Mixtec -mjd I L Northwest Maidu -mje I E Muskum -mjg I L Tu -mjh I L Mwera (Nyasa) -mji I L Kim Mun -mjj I L Mawak -mjk I L Matukar -mjl I L Mandeali -mjm I L Medebur -mjn I L Ma (Papua New Guinea) -mjo I L Malankuravan -mjp I L Malapandaram -mjq I E Malaryan -mjr I L Malavedan -mjs I L Miship -mjt I L Sauria Paharia -mju I L Manna-Dora -mjv I L Mannan -mjw I L Karbi -mjx I L Mahali -mjy I E Mahican -mjz I L Majhi -mka I L Mbre -mkb I L Mal Paharia -mkc I L Siliput -mkd mac mkd mk I L Macedonian -mke I L Mawchi -mkf I L Miya -mkg I L Mak (China) -mki I L Dhatki -mkj I L Mokilese -mkk I L Byep -mkl I L Mokole -mkm I L Moklen -mkn I L Kupang Malay -mko I L Mingang Doso -mkp I L Moikodi -mkq I E Bay Miwok -mkr I L Malas -mks I L Silacayoapan Mixtec -mkt I L Vamale -mku I L Konyanka Maninka -mkv I L Mafea -mkw I L Kituba (Congo) -mkx I L Kinamiging Manobo -mky I L East Makian -mkz I L Makasae -mla I L Malo -mlb I L Mbule -mlc I L Cao Lan -mle I L Manambu -mlf I L Mal -mlg mlg mlg mg M L Malagasy -mlh I L Mape -mli I L Malimpung -mlj I L Miltu -mlk I L Ilwana -mll I L Malua Bay -mlm I L Mulam -mln I L Malango -mlo I L Mlomp -mlp I L Bargam -mlq I L Western Maninkakan -mlr I L Vame -mls I L Masalit -mlt mlt mlt mt I L Maltese -mlu I L To'abaita -mlv I L Motlav -mlw I L Moloko -mlx I L Malfaxal -mlz I L Malaynon -mma I L Mama -mmb I L Momina -mmc I L Michoacán Mazahua -mmd I L Maonan -mme I L Mae -mmf I L Mundat -mmg I L North Ambrym -mmh I L Mehináku -mmi I L Musar -mmj I L Majhwar -mmk I L Mukha-Dora -mml I L Man Met -mmm I L Maii -mmn I L Mamanwa -mmo I L Mangga Buang -mmp I L Siawi -mmq I L Musak -mmr I L Western Xiangxi Miao -mmt I L Malalamai -mmu I L Mmaala -mmv I E Miriti -mmw I L Emae -mmx I L Madak -mmy I L Migaama -mmz I L Mabaale -mna I L Mbula -mnb I L Muna -mnc mnc mnc I L Manchu -mnd I L Mondé -mne I L Naba -mnf I L Mundani -mng I L Eastern Mnong -mnh I L Mono (Democratic Republic of Congo) -mni mni mni I L Manipuri -mnj I L Munji -mnk I L Mandinka -mnl I L Tiale -mnm I L Mapena -mnn I L Southern Mnong -mnp I L Min Bei Chinese -mnq I L Minriq -mnr I L Mono (USA) -mns I L Mansi -mnu I L Mer -mnv I L Rennell-Bellona -mnw I L Mon -mnx I L Manikion -mny I L Manyawa -mnz I L Moni -moa I L Mwan -moc I L Mocoví -mod I E Mobilian -moe I L Montagnais -mog I L Mongondow -moh moh moh I L Mohawk -moi I L Mboi -moj I L Monzombo -mok I L Morori -mom I E Mangue -mon mon mon mn M L Mongolian -moo I L Monom -mop I L Mopán Maya -moq I L Mor (Bomberai Peninsula) -mor I L Moro -mos mos mos I L Mossi -mot I L Barí -mou I L Mogum -mov I L Mohave -mow I L Moi (Congo) -mox I L Molima -moy I L Shekkacho -moz I L Mukulu -mpa I L Mpoto -mpb I L Mullukmulluk -mpc I L Mangarayi -mpd I L Machinere -mpe I L Majang -mpg I L Marba -mph I L Maung -mpi I L Mpade -mpj I L Martu Wangka -mpk I L Mbara (Chad) -mpl I L Middle Watut -mpm I L Yosondúa Mixtec -mpn I L Mindiri -mpo I L Miu -mpp I L Migabac -mpq I L Matís -mpr I L Vangunu -mps I L Dadibi -mpt I L Mian -mpu I L Makuráp -mpv I L Mungkip -mpw I L Mapidian -mpx I L Misima-Panaeati -mpy I L Mapia -mpz I L Mpi -mqa I L Maba (Indonesia) -mqb I L Mbuko -mqc I L Mangole -mqe I L Matepi -mqf I L Momuna -mqg I L Kota Bangun Kutai Malay -mqh I L Tlazoyaltepec Mixtec -mqi I L Mariri -mqj I L Mamasa -mqk I L Rajah Kabunsuwan Manobo -mql I L Mbelime -mqm I L South Marquesan -mqn I L Moronene -mqo I L Modole -mqp I L Manipa -mqq I L Minokok -mqr I L Mander -mqs I L West Makian -mqt I L Mok -mqu I L Mandari -mqv I L Mosimo -mqw I L Murupi -mqx I L Mamuju -mqy I L Manggarai -mqz I L Pano -mra I L Mlabri -mrb I L Marino -mrc I L Maricopa -mrd I L Western Magar -mre I E Martha's Vineyard Sign Language -mrf I L Elseng -mrg I L Mising -mrh I L Mara Chin -mri mao mri mi I L Maori -mrj I L Western Mari -mrk I L Hmwaveke -mrl I L Mortlockese -mrm I L Merlav -mrn I L Cheke Holo -mro I L Mru -mrp I L Morouas -mrq I L North Marquesan -mrr I L Maria (India) -mrs I L Maragus -mrt I L Marghi Central -mru I L Mono (Cameroon) -mrv I L Mangareva -mrw I L Maranao -mrx I L Maremgi -mry I L Mandaya -mrz I L Marind -msa may msa ms M L Malay (macrolanguage) -msb I L Masbatenyo -msc I L Sankaran Maninka -msd I L Yucatec Maya Sign Language -mse I L Musey -msf I L Mekwei -msg I L Moraid -msh I L Masikoro Malagasy -msi I L Sabah Malay -msj I L Ma (Democratic Republic of Congo) -msk I L Mansaka -msl I L Molof -msm I L Agusan Manobo -msn I L Vurës -mso I L Mombum -msp I E Maritsauá -msq I L Caac -msr I L Mongolian Sign Language -mss I L West Masela -msu I L Musom -msv I L Maslam -msw I L Mansoanka -msx I L Moresada -msy I L Aruamu -msz I L Momare -mta I L Cotabato Manobo -mtb I L Anyin Morofo -mtc I L Munit -mtd I L Mualang -mte I L Mono (Solomon Islands) -mtf I L Murik (Papua New Guinea) -mtg I L Una -mth I L Munggui -mti I L Maiwa (Papua New Guinea) -mtj I L Moskona -mtk I L Mbe' -mtl I L Montol -mtm I E Mator -mtn I E Matagalpa -mto I L Totontepec Mixe -mtp I L Wichí Lhamtés Nocten -mtq I L Muong -mtr I L Mewari -mts I L Yora -mtt I L Mota -mtu I L Tututepec Mixtec -mtv I L Asaro'o -mtw I L Southern Binukidnon -mtx I L Tidaá Mixtec -mty I L Nabi -mua I L Mundang -mub I L Mubi -muc I L Ajumbu -mud I L Mednyj Aleut -mue I L Media Lengua -mug I L Musgu -muh I L Mündü -mui I L Musi -muj I L Mabire -muk I L Mugom -mul mul mul S S Multiple languages -mum I L Maiwala -muo I L Nyong -mup I L Malvi -muq I L Eastern Xiangxi Miao -mur I L Murle -mus mus mus I L Creek -mut I L Western Muria -muu I L Yaaku -muv I L Muthuvan -mux I L Bo-Ung -muy I L Muyang -muz I L Mursi -mva I L Manam -mvb I E Mattole -mvd I L Mamboru -mve I L Marwari (Pakistan) -mvf I L Peripheral Mongolian -mvg I L Yucuañe Mixtec -mvh I L Mulgi -mvi I L Miyako -mvk I L Mekmek -mvl I E Mbara (Australia) -mvm I L Muya -mvn I L Minaveha -mvo I L Marovo -mvp I L Duri -mvq I L Moere -mvr I L Marau -mvs I L Massep -mvt I L Mpotovoro -mvu I L Marfa -mvv I L Tagal Murut -mvw I L Machinga -mvx I L Meoswar -mvy I L Indus Kohistani -mvz I L Mesqan -mwa I L Mwatebu -mwb I L Juwal -mwc I L Are -mwe I L Mwera (Chimwera) -mwf I L Murrinh-Patha -mwg I L Aiklep -mwh I L Mouk-Aria -mwi I L Labo -mwj I L Maligo -mwk I L Kita Maninkakan -mwl mwl mwl I L Mirandese -mwm I L Sar -mwn I L Nyamwanga -mwo I L Central Maewo -mwp I L Kala Lagaw Ya -mwq I L Mün Chin -mwr mwr mwr M L Marwari -mws I L Mwimbi-Muthambi -mwt I L Moken -mwu I E Mittu -mwv I L Mentawai -mww I L Hmong Daw -mwx I L Mediak -mwy I L Mosiro -mwz I L Moingi -mxa I L Northwest Oaxaca Mixtec -mxb I L Tezoatlán Mixtec -mxc I L Manyika -mxd I L Modang -mxe I L Mele-Fila -mxf I L Malgbe -mxg I L Mbangala -mxh I L Mvuba -mxi I E Mozarabic -mxj I L Miju-Mishmi -mxk I L Monumbo -mxl I L Maxi Gbe -mxm I L Meramera -mxn I L Moi (Indonesia) -mxo I L Mbowe -mxp I L Tlahuitoltepec Mixe -mxq I L Juquila Mixe -mxr I L Murik (Malaysia) -mxs I L Huitepec Mixtec -mxt I L Jamiltepec Mixtec -mxu I L Mada (Cameroon) -mxv I L Metlatónoc Mixtec -mxw I L Namo -mxx I L Mahou -mxy I L Southeastern Nochixtlán Mixtec -mxz I L Central Masela -mya bur mya my I L Burmese -myb I L Mbay -myc I L Mayeka -myd I L Maramba -mye I L Myene -myf I L Bambassi -myg I L Manta -myh I L Makah -myi I L Mina (India) -myj I L Mangayat -myk I L Mamara Senoufo -myl I L Moma -mym I L Me'en -myo I L Anfillo -myp I L Pirahã -myr I L Muniche -mys I E Mesmes -myu I L Mundurukú -myv myv myv I L Erzya -myw I L Muyuw -myx I L Masaaba -myy I L Macuna -myz I H Classical Mandaic -mza I L Santa María Zacatepec Mixtec -mzb I L Tumzabt -mzc I L Madagascar Sign Language -mzd I L Malimba -mze I L Morawa -mzg I L Monastic Sign Language -mzh I L Wichí Lhamtés Güisnay -mzi I L Ixcatlán Mazatec -mzj I L Manya -mzk I L Nigeria Mambila -mzl I L Mazatlán Mixe -mzm I L Mumuye -mzn I L Mazanderani -mzo I E Matipuhy -mzp I L Movima -mzq I L Mori Atas -mzr I L Marúbo -mzs I L Macanese -mzt I L Mintil -mzu I L Inapang -mzv I L Manza -mzw I L Deg -mzx I L Mawayana -mzy I L Mozambican Sign Language -mzz I L Maiadomu -naa I L Namla -nab I L Southern Nambikuára -nac I L Narak -nad I L Nijadali -nae I L Naka'ela -naf I L Nabak -nag I L Naga Pidgin -naj I L Nalu -nak I L Nakanai -nal I L Nalik -nam I L Ngan'gityemerri -nan I L Min Nan Chinese -nao I L Naaba -nap nap nap I L Neapolitan -naq I L Nama (Namibia) -nar I L Iguta -nas I L Naasioi -nat I L Hungworo -nau nau nau na I L Nauru -nav nav nav nv I L Navajo -naw I L Nawuri -nax I L Nakwi -nay I E Narrinyeri -naz I L Coatepec Nahuatl -nba I L Nyemba -nbb I L Ndoe -nbc I L Chang Naga -nbd I L Ngbinda -nbe I L Konyak Naga -nbg I L Nagarchal -nbh I L Ngamo -nbi I L Mao Naga -nbj I L Ngarinman -nbk I L Nake -nbl nbl nbl nr I L South Ndebele -nbm I L Ngbaka Ma'bo -nbn I L Kuri -nbo I L Nkukoli -nbp I L Nnam -nbq I L Nggem -nbr I L Numana-Nunku-Gbantu-Numbu -nbs I L Namibian Sign Language -nbt I L Na -nbu I L Rongmei Naga -nbv I L Ngamambo -nbw I L Southern Ngbandi -nby I L Ningera -nca I L Iyo -ncb I L Central Nicobarese -ncc I L Ponam -ncd I L Nachering -nce I L Yale -ncf I L Notsi -ncg I L Nisga'a -nch I L Central Huasteca Nahuatl -nci I H Classical Nahuatl -ncj I L Northern Puebla Nahuatl -nck I L Nakara -ncl I L Michoacán Nahuatl -ncm I L Nambo -ncn I L Nauna -nco I L Sibe -ncp I L Ndaktup -ncr I L Ncane -ncs I L Nicaraguan Sign Language -nct I L Chothe Naga -ncu I L Chumburung -ncx I L Central Puebla Nahuatl -ncz I E Natchez -nda I L Ndasa -ndb I L Kenswei Nsei -ndc I L Ndau -ndd I L Nde-Nsele-Nta -nde nde nde nd I L North Ndebele -ndf I E Nadruvian -ndg I L Ndengereko -ndh I L Ndali -ndi I L Samba Leko -ndj I L Ndamba -ndk I L Ndaka -ndl I L Ndolo -ndm I L Ndam -ndn I L Ngundi -ndo ndo ndo ng I L Ndonga -ndp I L Ndo -ndq I L Ndombe -ndr I L Ndoola -nds nds nds I L Low German -ndt I L Ndunga -ndu I L Dugun -ndv I L Ndut -ndw I L Ndobo -ndx I L Nduga -ndy I L Lutos -ndz I L Ndogo -nea I L Eastern Ngad'a -neb I L Toura (Côte d'Ivoire) -nec I L Nedebang -ned I L Nde-Gbite -nee I L Nêlêmwa-Nixumwak -nef I L Nefamese -neg I L Negidal -neh I L Nyenkha -nei I A Neo-Hittite -nej I L Neko -nek I L Neku -nem I L Nemi -nen I L Nengone -neo I L Ná-Meo -nep nep nep ne M L Nepali (macrolanguage) -neq I L North Central Mixe -ner I L Yahadian -nes I L Bhoti Kinnauri -net I L Nete -neu I C Neo -nev I L Nyaheun -new new new I L Newari -nex I L Neme -ney I L Neyo -nez I L Nez Perce -nfa I L Dhao -nfd I L Ahwai -nfl I L Ayiwo -nfr I L Nafaanra -nfu I L Mfumte -nga I L Ngbaka -ngb I L Northern Ngbandi -ngc I L Ngombe (Democratic Republic of Congo) -ngd I L Ngando (Central African Republic) -nge I L Ngemba -ngg I L Ngbaka Manza -ngh I L N/u -ngi I L Ngizim -ngj I L Ngie -ngk I L Dalabon -ngl I L Lomwe -ngm I L Ngatik Men's Creole -ngn I L Ngwo -ngo I L Ngoni -ngp I L Ngulu -ngq I L Ngurimi -ngr I L Engdewu -ngs I L Gvoko -ngt I L Ngeq -ngu I L Guerrero Nahuatl -ngv I E Nagumi -ngw I L Ngwaba -ngx I L Nggwahyi -ngy I L Tibea -ngz I L Ngungwel -nha I L Nhanda -nhb I L Beng -nhc I E Tabasco Nahuatl -nhd I L Chiripá -nhe I L Eastern Huasteca Nahuatl -nhf I L Nhuwala -nhg I L Tetelcingo Nahuatl -nhh I L Nahari -nhi I L Zacatlán-Ahuacatlán-Tepetzintla Nahuatl -nhk I L Isthmus-Cosoleacaque Nahuatl -nhm I L Morelos Nahuatl -nhn I L Central Nahuatl -nho I L Takuu -nhp I L Isthmus-Pajapan Nahuatl -nhq I L Huaxcaleca Nahuatl -nhr I L Naro -nht I L Ometepec Nahuatl -nhu I L Noone -nhv I L Temascaltepec Nahuatl -nhw I L Western Huasteca Nahuatl -nhx I L Isthmus-Mecayapan Nahuatl -nhy I L Northern Oaxaca Nahuatl -nhz I L Santa María La Alta Nahuatl -nia nia nia I L Nias -nib I L Nakame -nid I E Ngandi -nie I L Niellim -nif I L Nek -nig I E Ngalakan -nih I L Nyiha (Tanzania) -nii I L Nii -nij I L Ngaju -nik I L Southern Nicobarese -nil I L Nila -nim I L Nilamba -nin I L Ninzo -nio I L Nganasan -niq I L Nandi -nir I L Nimboran -nis I L Nimi -nit I L Southeastern Kolami -niu niu niu I L Niuean -niv I L Gilyak -niw I L Nimo -nix I L Hema -niy I L Ngiti -niz I L Ningil -nja I L Nzanyi -njb I L Nocte Naga -njd I L Ndonde Hamba -njh I L Lotha Naga -nji I L Gudanji -njj I L Njen -njl I L Njalgulgule -njm I L Angami Naga -njn I L Liangmai Naga -njo I L Ao Naga -njr I L Njerep -njs I L Nisa -njt I L Ndyuka-Trio Pidgin -nju I L Ngadjunmaya -njx I L Kunyi -njy I L Njyem -njz I L Nyishi -nka I L Nkoya -nkb I L Khoibu Naga -nkc I L Nkongho -nkd I L Koireng -nke I L Duke -nkf I L Inpui Naga -nkg I L Nekgini -nkh I L Khezha Naga -nki I L Thangal Naga -nkj I L Nakai -nkk I L Nokuku -nkm I L Namat -nkn I L Nkangala -nko I L Nkonya -nkp I E Niuatoputapu -nkq I L Nkami -nkr I L Nukuoro -nks I L North Asmat -nkt I L Nyika (Tanzania) -nku I L Bouna Kulango -nkv I L Nyika (Malawi and Zambia) -nkw I L Nkutu -nkx I L Nkoroo -nkz I L Nkari -nla I L Ngombale -nlc I L Nalca -nld dut nld nl I L Dutch -nle I L East Nyala -nlg I L Gela -nli I L Grangali -nlj I L Nyali -nlk I L Ninia Yali -nll I L Nihali -nlo I L Ngul -nlq I L Lao Naga -nlu I L Nchumbulu -nlv I L Orizaba Nahuatl -nlw I E Walangama -nlx I L Nahali -nly I L Nyamal -nlz I L Nalögo -nma I L Maram Naga -nmb I L Big Nambas -nmc I L Ngam -nmd I L Ndumu -nme I L Mzieme Naga -nmf I L Tangkhul Naga (India) -nmg I L Kwasio -nmh I L Monsang Naga -nmi I L Nyam -nmj I L Ngombe (Central African Republic) -nmk I L Namakura -nml I L Ndemli -nmm I L Manangba -nmn I L !Xóõ -nmo I L Moyon Naga -nmp I E Nimanbur -nmq I L Nambya -nmr I E Nimbari -nms I L Letemboi -nmt I L Namonuito -nmu I L Northeast Maidu -nmv I E Ngamini -nmw I L Nimoa -nmx I L Nama (Papua New Guinea) -nmy I L Namuyi -nmz I L Nawdm -nna I L Nyangumarta -nnb I L Nande -nnc I L Nancere -nnd I L West Ambae -nne I L Ngandyera -nnf I L Ngaing -nng I L Maring Naga -nnh I L Ngiemboon -nni I L North Nuaulu -nnj I L Nyangatom -nnk I L Nankina -nnl I L Northern Rengma Naga -nnm I L Namia -nnn I L Ngete -nno nno nno nn I L Norwegian Nynorsk -nnp I L Wancho Naga -nnq I L Ngindo -nnr I E Narungga -nns I L Ningye -nnt I E Nanticoke -nnu I L Dwang -nnv I E Nugunu (Australia) -nnw I L Southern Nuni -nnx I L Ngong -nny I E Nyangga -nnz I L Nda'nda' -noa I L Woun Meu -nob nob nob nb I L Norwegian Bokmål -noc I L Nuk -nod I L Northern Thai -noe I L Nimadi -nof I L Nomane -nog nog nog I L Nogai -noh I L Nomu -noi I L Noiri -noj I L Nonuya -nok I E Nooksack -nol I E Nomlaki -nom I E Nocamán -non non non I H Old Norse -nop I L Numanggang -noq I L Ngongo -nor nor nor no M L Norwegian -nos I L Eastern Nisu -not I L Nomatsiguenga -nou I L Ewage-Notu -nov I C Novial -now I L Nyambo -noy I L Noy -noz I L Nayi -npa I L Nar Phu -npb I L Nupbikha -npg I L Ponyo-Gongwang Naga -nph I L Phom Naga -npi I L Nepali (individual language) -npl I L Southeastern Puebla Nahuatl -npn I L Mondropolon -npo I L Pochuri Naga -nps I L Nipsan -npu I L Puimei Naga -npy I L Napu -nqg I L Southern Nago -nqk I L Kura Ede Nago -nqm I L Ndom -nqn I L Nen -nqo nqo nqo I L N'Ko -nqq I L Kyan-Karyaw Naga -nqy I L Akyaung Ari Naga -nra I L Ngom -nrb I L Nara -nrc I A Noric -nre I L Southern Rengma Naga -nrg I L Narango -nri I L Chokri Naga -nrk I L Ngarla -nrl I L Ngarluma -nrm I L Narom -nrn I E Norn -nrp I A North Picene -nrr I E Norra -nrt I E Northern Kalapuya -nru I L Narua -nrx I E Ngurmbur -nrz I L Lala -nsa I L Sangtam Naga -nsc I L Nshi -nsd I L Southern Nisu -nse I L Nsenga -nsf I L Northwestern Nisu -nsg I L Ngasa -nsh I L Ngoshie -nsi I L Nigerian Sign Language -nsk I L Naskapi -nsl I L Norwegian Sign Language -nsm I L Sumi Naga -nsn I L Nehan -nso nso nso I L Pedi -nsp I L Nepalese Sign Language -nsq I L Northern Sierra Miwok -nsr I L Maritime Sign Language -nss I L Nali -nst I L Tase Naga -nsu I L Sierra Negra Nahuatl -nsv I L Southwestern Nisu -nsw I L Navut -nsx I L Nsongo -nsy I L Nasal -nsz I L Nisenan -nte I L Nathembo -ntg I E Ngantangarra -nti I L Natioro -ntj I L Ngaanyatjarra -ntk I L Ikoma-Nata-Isenye -ntm I L Nateni -nto I L Ntomba -ntp I L Northern Tepehuan -ntr I L Delo -nts I E Natagaimas -ntu I L Natügu -ntw I E Nottoway -ntx I L Tangkhul Naga (Myanmar) -nty I L Mantsi -ntz I L Natanzi -nua I L Yuanga -nuc I E Nukuini -nud I L Ngala -nue I L Ngundu -nuf I L Nusu -nug I E Nungali -nuh I L Ndunda -nui I L Ngumbi -nuj I L Nyole -nuk I L Nuu-chah-nulth -nul I L Nusa Laut -num I L Niuafo'ou -nun I L Anong -nuo I L Nguôn -nup I L Nupe-Nupe-Tako -nuq I L Nukumanu -nur I L Nukuria -nus I L Nuer -nut I L Nung (Viet Nam) -nuu I L Ngbundu -nuv I L Northern Nuni -nuw I L Nguluwan -nux I L Mehek -nuy I L Nunggubuyu -nuz I L Tlamacazapa Nahuatl -nvh I L Nasarian -nvm I L Namiae -nvo I L Nyokon -nwa I E Nawathinehena -nwb I L Nyabwa -nwc nwc nwc I H Classical Newari -nwe I L Ngwe -nwg I E Ngayawung -nwi I L Southwest Tanna -nwm I L Nyamusa-Molo -nwo I E Nauo -nwr I L Nawaru -nwx I H Middle Newar -nwy I E Nottoway-Meherrin -nxa I L Nauete -nxd I L Ngando (Democratic Republic of Congo) -nxe I L Nage -nxg I L Ngad'a -nxi I L Nindi -nxk I L Koki Naga -nxl I L South Nuaulu -nxm I A Numidian -nxn I E Ngawun -nxq I L Naxi -nxr I L Ninggerum -nxu I E Narau -nxx I L Nafri -nya nya nya ny I L Nyanja -nyb I L Nyangbo -nyc I L Nyanga-li -nyd I L Nyore -nye I L Nyengo -nyf I L Giryama -nyg I L Nyindu -nyh I L Nyigina -nyi I L Ama (Sudan) -nyj I L Nyanga -nyk I L Nyaneka -nyl I L Nyeu -nym nym nym I L Nyamwezi -nyn nyn nyn I L Nyankole -nyo nyo nyo I L Nyoro -nyp I E Nyang'i -nyq I L Nayini -nyr I L Nyiha (Malawi) -nys I L Nyunga -nyt I E Nyawaygi -nyu I L Nyungwe -nyv I E Nyulnyul -nyw I L Nyaw -nyx I E Nganyaywana -nyy I L Nyakyusa-Ngonde -nza I L Tigon Mbembe -nzb I L Njebi -nzi nzi nzi I L Nzima -nzk I L Nzakara -nzm I L Zeme Naga -nzs I L New Zealand Sign Language -nzu I L Teke-Nzikou -nzy I L Nzakambay -nzz I L Nanga Dama Dogon -oaa I L Orok -oac I L Oroch -oar I A Old Aramaic (up to 700 BCE) -oav I H Old Avar -obi I E Obispeño -obk I L Southern Bontok -obl I L Oblo -obm I A Moabite -obo I L Obo Manobo -obr I H Old Burmese -obt I H Old Breton -obu I L Obulom -oca I L Ocaina -och I A Old Chinese -oci oci oci oc I L Occitan (post 1500) -oco I H Old Cornish -ocu I L Atzingo Matlatzinca -oda I L Odut -odk I L Od -odt I H Old Dutch -odu I L Odual -ofo I E Ofo -ofs I H Old Frisian -ofu I L Efutop -ogb I L Ogbia -ogc I L Ogbah -oge I H Old Georgian -ogg I L Ogbogolo -ogo I L Khana -ogu I L Ogbronuagum -oht I A Old Hittite -ohu I H Old Hungarian -oia I L Oirata -oin I L Inebu One -ojb I L Northwestern Ojibwa -ojc I L Central Ojibwa -ojg I L Eastern Ojibwa -oji oji oji oj M L Ojibwa -ojp I H Old Japanese -ojs I L Severn Ojibwa -ojv I L Ontong Java -ojw I L Western Ojibwa -oka I L Okanagan -okb I L Okobo -okd I L Okodia -oke I L Okpe (Southwestern Edo) -okg I E Koko Babangk -okh I L Koresh-e Rostam -oki I L Okiek -okj I E Oko-Juwoi -okk I L Kwamtim One -okl I E Old Kentish Sign Language -okm I H Middle Korean (10th-16th cent.) -okn I L Oki-No-Erabu -oko I H Old Korean (3rd-9th cent.) -okr I L Kirike -oks I L Oko-Eni-Osayen -oku I L Oku -okv I L Orokaiva -okx I L Okpe (Northwestern Edo) -ola I L Walungge -old I L Mochi -ole I L Olekha -olk I E Olkol -olm I L Oloma -olo I L Livvi -olr I L Olrat -oma I L Omaha-Ponca -omb I L East Ambae -omc I E Mochica -ome I E Omejes -omg I L Omagua -omi I L Omi -omk I E Omok -oml I L Ombo -omn I A Minoan -omo I L Utarmbung -omp I H Old Manipuri -omr I H Old Marathi -omt I L Omotik -omu I E Omurano -omw I L South Tairora -omx I H Old Mon -ona I L Ona -onb I L Lingao -one I L Oneida -ong I L Olo -oni I L Onin -onj I L Onjob -onk I L Kabore One -onn I L Onobasulu -ono I L Onondaga -onp I L Sartang -onr I L Northern One -ons I L Ono -ont I L Ontenu -onu I L Unua -onw I H Old Nubian -onx I L Onin Based Pidgin -ood I L Tohono O'odham -oog I L Ong -oon I L Önge -oor I L Oorlams -oos I A Old Ossetic -opa I L Okpamheri -opk I L Kopkaka -opm I L Oksapmin -opo I L Opao -opt I E Opata -opy I L Ofayé -ora I L Oroha -orc I L Orma -ore I L Orejón -org I L Oring -orh I L Oroqen -ori ori ori or M L Oriya (macrolanguage) -orm orm orm om M L Oromo -orn I L Orang Kanaq -oro I L Orokolo -orr I L Oruma -ors I L Orang Seletar -ort I L Adivasi Oriya -oru I L Ormuri -orv I H Old Russian -orw I L Oro Win -orx I L Oro -ory I L Oriya (individual language) -orz I L Ormu -osa osa osa I L Osage -osc I A Oscan -osi I L Osing -oso I L Ososo -osp I H Old Spanish -oss oss oss os I L Ossetian -ost I L Osatu -osu I L Southern One -osx I H Old Saxon -ota ota ota I H Ottoman Turkish (1500-1928) -otb I H Old Tibetan -otd I L Ot Danum -ote I L Mezquital Otomi -oti I E Oti -otk I H Old Turkish -otl I L Tilapa Otomi -otm I L Eastern Highland Otomi -otn I L Tenango Otomi -otq I L Querétaro Otomi -otr I L Otoro -ots I L Estado de México Otomi -ott I L Temoaya Otomi -otu I E Otuke -otw I L Ottawa -otx I L Texcatepec Otomi -oty I A Old Tamil -otz I L Ixtenco Otomi -oua I L Tagargrent -oub I L Glio-Oubi -oue I L Oune -oui I H Old Uighur -oum I E Ouma -oun I L !O!ung -owi I L Owiniga -owl I H Old Welsh -oyb I L Oy -oyd I L Oyda -oym I L Wayampi -oyy I L Oya'oya -ozm I L Koonzime -pab I L Parecís -pac I L Pacoh -pad I L Paumarí -pae I L Pagibete -paf I E Paranawát -pag pag pag I L Pangasinan -pah I L Tenharim -pai I L Pe -pak I L Parakanã -pal pal pal I A Pahlavi -pam pam pam I L Pampanga -pan pan pan pa I L Panjabi -pao I L Northern Paiute -pap pap pap I L Papiamento -paq I L Parya -par I L Panamint -pas I L Papasena -pat I L Papitalai -pau pau pau I L Palauan -pav I L Pakaásnovos -paw I L Pawnee -pax I E Pankararé -pay I L Pech -paz I E Pankararú -pbb I L Páez -pbc I L Patamona -pbe I L Mezontla Popoloca -pbf I L Coyotepec Popoloca -pbg I E Paraujano -pbh I L E'ñapa Woromaipu -pbi I L Parkwa -pbl I L Mak (Nigeria) -pbn I L Kpasam -pbo I L Papel -pbp I L Badyara -pbr I L Pangwa -pbs I L Central Pame -pbt I L Southern Pashto -pbu I L Northern Pashto -pbv I L Pnar -pby I L Pyu -pca I L Santa Inés Ahuatempan Popoloca -pcb I L Pear -pcc I L Bouyei -pcd I L Picard -pce I L Ruching Palaung -pcf I L Paliyan -pcg I L Paniya -pch I L Pardhan -pci I L Duruwa -pcj I L Parenga -pck I L Paite Chin -pcl I L Pardhi -pcm I L Nigerian Pidgin -pcn I L Piti -pcp I L Pacahuara -pcw I L Pyapun -pda I L Anam -pdc I L Pennsylvania German -pdi I L Pa Di -pdn I L Podena -pdo I L Padoe -pdt I L Plautdietsch -pdu I L Kayan -pea I L Peranakan Indonesian -peb I E Eastern Pomo -ped I L Mala (Papua New Guinea) -pee I L Taje -pef I E Northeastern Pomo -peg I L Pengo -peh I L Bonan -pei I L Chichimeca-Jonaz -pej I E Northern Pomo -pek I L Penchal -pel I L Pekal -pem I L Phende -peo peo peo I H Old Persian (ca. 600-400 B.C.) -pep I L Kunja -peq I L Southern Pomo -pes I L Iranian Persian -pev I L Pémono -pex I L Petats -pey I L Petjo -pez I L Eastern Penan -pfa I L Pááfang -pfe I L Peere -pfl I L Pfaelzisch -pga I L Sudanese Creole Arabic -pgg I L Pangwali -pgi I L Pagi -pgk I L Rerep -pgl I A Primitive Irish -pgn I A Paelignian -pgs I L Pangseng -pgu I L Pagu -pha I L Pa-Hng -phd I L Phudagi -phg I L Phuong -phh I L Phukha -phk I L Phake -phl I L Phalura -phm I L Phimbi -phn phn phn I A Phoenician -pho I L Phunoi -phq I L Phana' -phr I L Pahari-Potwari -pht I L Phu Thai -phu I L Phuan -phv I L Pahlavani -phw I L Phangduwali -pia I L Pima Bajo -pib I L Yine -pic I L Pinji -pid I L Piaroa -pie I E Piro -pif I L Pingelapese -pig I L Pisabo -pih I L Pitcairn-Norfolk -pii I L Pini -pij I E Pijao -pil I L Yom -pim I E Powhatan -pin I L Piame -pio I L Piapoco -pip I L Pero -pir I L Piratapuyo -pis I L Pijin -pit I E Pitta Pitta -piu I L Pintupi-Luritja -piv I L Pileni -piw I L Pimbwe -pix I L Piu -piy I L Piya-Kwonci -piz I L Pije -pjt I L Pitjantjatjara -pka I H Ardhamāgadhī Prākrit -pkb I L Pokomo -pkc I E Paekche -pkg I L Pak-Tong -pkh I L Pankhu -pkn I L Pakanha -pko I L Pökoot -pkp I L Pukapuka -pkr I L Attapady Kurumba -pks I L Pakistan Sign Language -pkt I L Maleng -pku I L Paku -pla I L Miani -plb I L Polonombauk -plc I L Central Palawano -pld I L Polari -ple I L Palu'e -plg I L Pilagá -plh I L Paulohi -pli pli pli pi I A Pali -plj I L Polci -plk I L Kohistani Shina -pll I L Shwe Palaung -pln I L Palenquero -plo I L Oluta Popoluca -plp I L Palpa -plq I A Palaic -plr I L Palaka Senoufo -pls I L San Marcos Tlalcoyalco Popoloca -plt I L Plateau Malagasy -plu I L Palikúr -plv I L Southwest Palawano -plw I L Brooke's Point Palawano -ply I L Bolyu -plz I L Paluan -pma I L Paama -pmb I L Pambia -pmc I E Palumata -pmd I E Pallanganmiddang -pme I L Pwaamei -pmf I L Pamona -pmh I H Māhārāṣṭri Prākrit -pmi I L Northern Pumi -pmj I L Southern Pumi -pmk I E Pamlico -pml I E Lingua Franca -pmm I L Pomo -pmn I L Pam -pmo I L Pom -pmq I L Northern Pame -pmr I L Paynamar -pms I L Piemontese -pmt I L Tuamotuan -pmu I L Mirpur Panjabi -pmw I L Plains Miwok -pmx I L Poumei Naga -pmy I L Papuan Malay -pmz I E Southern Pame -pna I L Punan Bah-Biau -pnb I L Western Panjabi -pnc I L Pannei -pne I L Western Penan -png I L Pongu -pnh I L Penrhyn -pni I L Aoheng -pnj I E Pinjarup -pnk I L Paunaka -pnl I L Paleni -pnm I L Punan Batu 1 -pnn I L Pinai-Hagahai -pno I E Panobo -pnp I L Pancana -pnq I L Pana (Burkina Faso) -pnr I L Panim -pns I L Ponosakan -pnt I L Pontic -pnu I L Jiongnai Bunu -pnv I L Pinigura -pnw I L Panytyima -pnx I L Phong-Kniang -pny I L Pinyin -pnz I L Pana (Central African Republic) -poc I L Poqomam -pod I E Ponares -poe I L San Juan Atzingo Popoloca -pof I L Poke -pog I E Potiguára -poh I L Poqomchi' -poi I L Highland Popoluca -pok I L Pokangá -pol pol pol pl I L Polish -pom I L Southeastern Pomo -pon pon pon I L Pohnpeian -poo I L Central Pomo -pop I L Pwapwâ -poq I L Texistepec Popoluca -por por por pt I L Portuguese -pos I L Sayula Popoluca -pot I L Potawatomi -pov I L Upper Guinea Crioulo -pow I L San Felipe Otlaltepec Popoloca -pox I E Polabian -poy I L Pogolo -ppa I L Pao -ppe I L Papi -ppi I L Paipai -ppk I L Uma -ppl I L Pipil -ppm I L Papuma -ppn I L Papapana -ppo I L Folopa -ppp I L Pelende -ppq I L Pei -pps I L San Luís Temalacayuca Popoloca -ppt I L Pare -ppu I E Papora -pqa I L Pa'a -pqm I L Malecite-Passamaquoddy -prb I L Lua' -prc I L Parachi -prd I L Parsi-Dari -pre I L Principense -prf I L Paranan -prg I L Prussian -prh I L Porohanon -pri I L Paicî -prk I L Parauk -prl I L Peruvian Sign Language -prm I L Kibiri -prn I L Prasuni -pro pro pro I H Old Provençal (to 1500) -prp I L Parsi -prq I L Ashéninka Perené -prr I E Puri -prs I L Dari -prt I L Phai -pru I L Puragi -prw I L Parawen -prx I L Purik -pry I L Pray 3 -prz I L Providencia Sign Language -psa I L Asue Awyu -psc I L Persian Sign Language -psd I L Plains Indian Sign Language -pse I L Central Malay -psg I L Penang Sign Language -psh I L Southwest Pashayi -psi I L Southeast Pashayi -psl I L Puerto Rican Sign Language -psm I E Pauserna -psn I L Panasuan -pso I L Polish Sign Language -psp I L Philippine Sign Language -psq I L Pasi -psr I L Portuguese Sign Language -pss I L Kaulong -pst I L Central Pashto -psu I H Sauraseni Prākrit -psw I L Port Sandwich -psy I E Piscataway -pta I L Pai Tavytera -pth I E Pataxó Hã-Ha-Hãe -pti I L Pintiini -ptn I L Patani -pto I L Zo'é -ptp I L Patep -ptr I L Piamatsina -ptt I L Enrekang -ptu I L Bambam -ptv I L Port Vato -ptw I E Pentlatch -pty I L Pathiya -pua I L Western Highland Purepecha -pub I L Purum -puc I L Punan Merap -pud I L Punan Aput -pue I L Puelche -puf I L Punan Merah -pug I L Phuie -pui I L Puinave -puj I L Punan Tubu -puk I L Pu Ko -pum I L Puma -puo I L Puoc -pup I L Pulabu -puq I E Puquina -pur I L Puruborá -pus pus pus ps M L Pushto -put I L Putoh -puu I L Punu -puw I L Puluwatese -pux I L Puare -puy I E Purisimeño -puz I L Purum Naga -pwa I L Pawaia -pwb I L Panawa -pwg I L Gapapaiwa -pwi I E Patwin -pwm I L Molbog -pwn I L Paiwan -pwo I L Pwo Western Karen -pwr I L Powari -pww I L Pwo Northern Karen -pxm I L Quetzaltepec Mixe -pye I L Pye Krumen -pym I L Fyam -pyn I L Poyanáwa -pys I L Paraguayan Sign Language -pyu I L Puyuma -pyx I A Pyu (Myanmar) -pyy I L Pyen -pzn I L Para Naga -qua I L Quapaw -qub I L Huallaga Huánuco Quechua -quc I L K'iche' -qud I L Calderón Highland Quichua -que que que qu M L Quechua -quf I L Lambayeque Quechua -qug I L Chimborazo Highland Quichua -quh I L South Bolivian Quechua -qui I L Quileute -quk I L Chachapoyas Quechua -qul I L North Bolivian Quechua -qum I L Sipacapense -qun I E Quinault -qup I L Southern Pastaza Quechua -quq I L Quinqui -qur I L Yanahuanca Pasco Quechua -qus I L Santiago del Estero Quichua -quv I L Sacapulteco -quw I L Tena Lowland Quichua -qux I L Yauyos Quechua -quy I L Ayacucho Quechua -quz I L Cusco Quechua -qva I L Ambo-Pasco Quechua -qvc I L Cajamarca Quechua -qve I L Eastern Apurímac Quechua -qvh I L Huamalíes-Dos de Mayo Huánuco Quechua -qvi I L Imbabura Highland Quichua -qvj I L Loja Highland Quichua -qvl I L Cajatambo North Lima Quechua -qvm I L Margos-Yarowilca-Lauricocha Quechua -qvn I L North Junín Quechua -qvo I L Napo Lowland Quechua -qvp I L Pacaraos Quechua -qvs I L San Martín Quechua -qvw I L Huaylla Wanca Quechua -qvy I L Queyu -qvz I L Northern Pastaza Quichua -qwa I L Corongo Ancash Quechua -qwc I H Classical Quechua -qwh I L Huaylas Ancash Quechua -qwm I E Kuman (Russia) -qws I L Sihuas Ancash Quechua -qwt I E Kwalhioqua-Tlatskanai -qxa I L Chiquián Ancash Quechua -qxc I L Chincha Quechua -qxh I L Panao Huánuco Quechua -qxl I L Salasaca Highland Quichua -qxn I L Northern Conchucos Ancash Quechua -qxo I L Southern Conchucos Ancash Quechua -qxp I L Puno Quechua -qxq I L Qashqa'i -qxr I L Cañar Highland Quichua -qxs I L Southern Qiang -qxt I L Santa Ana de Tusi Pasco Quechua -qxu I L Arequipa-La Unión Quechua -qxw I L Jauja Wanca Quechua -qya I C Quenya -qyp I E Quiripi -raa I L Dungmali -rab I L Camling -rac I L Rasawa -rad I L Rade -raf I L Western Meohang -rag I L Logooli -rah I L Rabha -rai I L Ramoaaina -raj raj raj M L Rajasthani -rak I L Tulu-Bohuai -ral I L Ralte -ram I L Canela -ran I L Riantana -rao I L Rao -rap rap rap I L Rapanui -raq I L Saam -rar rar rar I L Rarotongan -ras I L Tegali -rat I L Razajerdi -rau I L Raute -rav I L Sampang -raw I L Rawang -rax I L Rang -ray I L Rapa -raz I L Rahambuu -rbb I L Rumai Palaung -rbk I L Northern Bontok -rbl I L Miraya Bikol -rbp I E Barababaraba -rcf I L Réunion Creole French -rdb I L Rudbari -rea I L Rerau -reb I L Rembong -ree I L Rejang Kayan -reg I L Kara (Tanzania) -rei I L Reli -rej I L Rejang -rel I L Rendille -rem I E Remo -ren I L Rengao -rer I E Rer Bare -res I L Reshe -ret I L Retta -rey I L Reyesano -rga I L Roria -rge I L Romano-Greek -rgk I E Rangkas -rgn I L Romagnol -rgr I L Resígaro -rgs I L Southern Roglai -rgu I L Ringgou -rhg I L Rohingya -rhp I L Yahang -ria I L Riang (India) -rie I L Rien -rif I L Tarifit -ril I L Riang (Myanmar) -rim I L Nyaturu -rin I L Nungu -rir I L Ribun -rit I L Ritarungo -riu I L Riung -rjg I L Rajong -rji I L Raji -rjs I L Rajbanshi -rka I L Kraol -rkb I L Rikbaktsa -rkh I L Rakahanga-Manihiki -rki I L Rakhine -rkm I L Marka -rkt I L Rangpuri -rkw I E Arakwal -rma I L Rama -rmb I L Rembarunga -rmc I L Carpathian Romani -rmd I E Traveller Danish -rme I L Angloromani -rmf I L Kalo Finnish Romani -rmg I L Traveller Norwegian -rmh I L Murkim -rmi I L Lomavren -rmk I L Romkun -rml I L Baltic Romani -rmm I L Roma -rmn I L Balkan Romani -rmo I L Sinte Romani -rmp I L Rempi -rmq I L Caló -rms I L Romanian Sign Language -rmt I L Domari -rmu I L Tavringer Romani -rmv I C Romanova -rmw I L Welsh Romani -rmx I L Romam -rmy I L Vlax Romani -rmz I L Marma -rna I E Runa -rnd I L Ruund -rng I L Ronga -rnl I L Ranglong -rnn I L Roon -rnp I L Rongpo -rnr I E Nari Nari -rnw I L Rungwa -rob I L Tae' -roc I L Cacgia Roglai -rod I L Rogo -roe I L Ronji -rof I L Rombo -rog I L Northern Roglai -roh roh roh rm I L Romansh -rol I L Romblomanon -rom rom rom M L Romany -ron rum ron ro I L Romanian -roo I L Rotokas -rop I L Kriol -ror I L Rongga -rou I L Runga -row I L Dela-Oenale -rpn I L Repanbitip -rpt I L Rapting -rri I L Ririo -rro I L Waima -rrt I E Arritinngithigh -rsb I L Romano-Serbian -rsi I L Rennellese Sign Language -rsl I L Russian Sign Language -rtc I L Rungtu Chin -rth I L Ratahan -rtm I L Rotuman -rtw I L Rathawi -rub I L Gungu -ruc I L Ruuli -rue I L Rusyn -ruf I L Luguru -rug I L Roviana -ruh I L Ruga -rui I L Rufiji -ruk I L Che -run run run rn I L Rundi -ruo I L Istro Romanian -rup rup rup I L Macedo-Romanian -ruq I L Megleno Romanian -rus rus rus ru I L Russian -rut I L Rutul -ruu I L Lanas Lobu -ruy I L Mala (Nigeria) -ruz I L Ruma -rwa I L Rawo -rwk I L Rwa -rwm I L Amba (Uganda) -rwo I L Rawa -rwr I L Marwari (India) -rxd I L Ngardi -rxw I E Karuwali -ryn I L Northern Amami-Oshima -rys I L Yaeyama -ryu I L Central Okinawan -saa I L Saba -sab I L Buglere -sac I L Meskwaki -sad sad sad I L Sandawe -sae I L Sabanê -saf I L Safaliba -sag sag sag sg I L Sango -sah sah sah I L Yakut -saj I L Sahu -sak I L Sake -sam sam sam I E Samaritan Aramaic -san san san sa I A Sanskrit -sao I L Sause -sap I L Sanapaná -saq I L Samburu -sar I E Saraveca -sas sas sas I L Sasak -sat sat sat I L Santali -sau I L Saleman -sav I L Saafi-Saafi -saw I L Sawi -sax I L Sa -say I L Saya -saz I L Saurashtra -sba I L Ngambay -sbb I L Simbo -sbc I L Kele (Papua New Guinea) -sbd I L Southern Samo -sbe I L Saliba -sbf I L Shabo -sbg I L Seget -sbh I L Sori-Harengan -sbi I L Seti -sbj I L Surbakhal -sbk I L Safwa -sbl I L Botolan Sambal -sbm I L Sagala -sbn I L Sindhi Bhil -sbo I L Sabüm -sbp I L Sangu (Tanzania) -sbq I L Sileibi -sbr I L Sembakung Murut -sbs I L Subiya -sbt I L Kimki -sbu I L Stod Bhoti -sbv I A Sabine -sbw I L Simba -sbx I L Seberuang -sby I L Soli -sbz I L Sara Kaba -scb I L Chut -sce I L Dongxiang -scf I L San Miguel Creole French -scg I L Sanggau -sch I L Sakachep -sci I L Sri Lankan Creole Malay -sck I L Sadri -scl I L Shina -scn scn scn I L Sicilian -sco sco sco I L Scots -scp I L Helambu Sherpa -scq I L Sa'och -scs I L North Slavey -scu I L Shumcho -scv I L Sheni -scw I L Sha -scx I A Sicel -sda I L Toraja-Sa'dan -sdb I L Shabak -sdc I L Sassarese Sardinian -sde I L Surubu -sdf I L Sarli -sdg I L Savi -sdh I L Southern Kurdish -sdj I L Suundi -sdk I L Sos Kundi -sdl I L Saudi Arabian Sign Language -sdm I L Semandang -sdn I L Gallurese Sardinian -sdo I L Bukar-Sadung Bidayuh -sdp I L Sherdukpen -sdr I L Oraon Sadri -sds I E Sened -sdt I E Shuadit -sdu I L Sarudu -sdx I L Sibu Melanau -sdz I L Sallands -sea I L Semai -seb I L Shempire Senoufo -sec I L Sechelt -sed I L Sedang -see I L Seneca -sef I L Cebaara Senoufo -seg I L Segeju -seh I L Sena -sei I L Seri -sej I L Sene -sek I L Sekani -sel sel sel I L Selkup -sen I L Nanerigé Sénoufo -seo I L Suarmin -sep I L Sìcìté Sénoufo -seq I L Senara Sénoufo -ser I L Serrano -ses I L Koyraboro Senni Songhai -set I L Sentani -seu I L Serui-Laut -sev I L Nyarafolo Senoufo -sew I L Sewa Bay -sey I L Secoya -sez I L Senthang Chin -sfb I L Langue des signes de Belgique Francophone -sfe I L Eastern Subanen -sfm I L Small Flowery Miao -sfs I L South African Sign Language -sfw I L Sehwi -sga sga sga I H Old Irish (to 900) -sgb I L Mag-antsi Ayta -sgc I L Kipsigis -sgd I L Surigaonon -sge I L Segai -sgg I L Swiss-German Sign Language -sgh I L Shughni -sgi I L Suga -sgj I L Surgujia -sgk I L Sangkong -sgm I E Singa -sgo I L Songa -sgp I L Singpho -sgr I L Sangisari -sgs I L Samogitian -sgt I L Brokpake -sgu I L Salas -sgw I L Sebat Bet Gurage -sgx I L Sierra Leone Sign Language -sgy I L Sanglechi -sgz I L Sursurunga -sha I L Shall-Zwall -shb I L Ninam -shc I L Sonde -shd I L Kundal Shahi -she I L Sheko -shg I L Shua -shh I L Shoshoni -shi I L Tachelhit -shj I L Shatt -shk I L Shilluk -shl I L Shendu -shm I L Shahrudi -shn shn shn I L Shan -sho I L Shanga -shp I L Shipibo-Conibo -shq I L Sala -shr I L Shi -shs I L Shuswap -sht I E Shasta -shu I L Chadian Arabic -shv I L Shehri -shw I L Shwai -shx I L She -shy I L Tachawit -shz I L Syenara Senoufo -sia I E Akkala Sami -sib I L Sebop -sid sid sid I L Sidamo -sie I L Simaa -sif I L Siamou -sig I L Paasaal -sih I L Zire -sii I L Shom Peng -sij I L Numbami -sik I L Sikiana -sil I L Tumulung Sisaala -sim I L Mende (Papua New Guinea) -sin sin sin si I L Sinhala -sip I L Sikkimese -siq I L Sonia -sir I L Siri -sis I E Siuslaw -siu I L Sinagen -siv I L Sumariup -siw I L Siwai -six I L Sumau -siy I L Sivandi -siz I L Siwi -sja I L Epena -sjb I L Sajau Basap -sjd I L Kildin Sami -sje I L Pite Sami -sjg I L Assangori -sjk I E Kemi Sami -sjl I L Sajalong -sjm I L Mapun -sjn I C Sindarin -sjo I L Xibe -sjp I L Surjapuri -sjr I L Siar-Lak -sjs I E Senhaja De Srair -sjt I L Ter Sami -sju I L Ume Sami -sjw I L Shawnee -ska I L Skagit -skb I L Saek -skc I L Ma Manda -skd I L Southern Sierra Miwok -ske I L Seke (Vanuatu) -skf I L Sakirabiá -skg I L Sakalava Malagasy -skh I L Sikule -ski I L Sika -skj I L Seke (Nepal) -skk I L Sok -skm I L Kutong -skn I L Kolibugan Subanon -sko I L Seko Tengah -skp I L Sekapan -skq I L Sininkere -skr I L Seraiki -sks I L Maia -skt I L Sakata -sku I L Sakao -skv I L Skou -skw I E Skepi Creole Dutch -skx I L Seko Padang -sky I L Sikaiana -skz I L Sekar -slc I L Sáliba -sld I L Sissala -sle I L Sholaga -slf I L Swiss-Italian Sign Language -slg I L Selungai Murut -slh I L Southern Puget Sound Salish -sli I L Lower Silesian -slj I L Salumá -slk slo slk sk I L Slovak -sll I L Salt-Yui -slm I L Pangutaran Sama -sln I E Salinan -slp I L Lamaholot -slq I L Salchuq -slr I L Salar -sls I L Singapore Sign Language -slt I L Sila -slu I L Selaru -slv slv slv sl I L Slovenian -slw I L Sialum -slx I L Salampasu -sly I L Selayar -slz I L Ma'ya -sma sma sma I L Southern Sami -smb I L Simbari -smc I E Som -smd I L Sama -sme sme sme se I L Northern Sami -smf I L Auwe -smg I L Simbali -smh I L Samei -smj smj smj I L Lule Sami -smk I L Bolinao -sml I L Central Sama -smm I L Musasa -smn smn smn I L Inari Sami -smo smo smo sm I L Samoan -smp I E Samaritan -smq I L Samo -smr I L Simeulue -sms sms sms I L Skolt Sami -smt I L Simte -smu I E Somray -smv I L Samvedi -smw I L Sumbawa -smx I L Samba -smy I L Semnani -smz I L Simeku -sna sna sna sn I L Shona -snb I L Sebuyau -snc I L Sinaugoro -snd snd snd sd I L Sindhi -sne I L Bau Bidayuh -snf I L Noon -sng I L Sanga (Democratic Republic of Congo) -snh I E Shinabo -sni I E Sensi -snj I L Riverain Sango -snk snk snk I L Soninke -snl I L Sangil -snm I L Southern Ma'di -snn I L Siona -sno I L Snohomish -snp I L Siane -snq I L Sangu (Gabon) -snr I L Sihan -sns I L South West Bay -snu I L Senggi -snv I L Sa'ban -snw I L Selee -snx I L Sam -sny I L Saniyo-Hiyewe -snz I L Sinsauru -soa I L Thai Song -sob I L Sobei -soc I L So (Democratic Republic of Congo) -sod I L Songoora -soe I L Songomeno -sog sog sog I A Sogdian -soh I L Aka -soi I L Sonha -soj I L Soi -sok I L Sokoro -sol I L Solos -som som som so I L Somali -soo I L Songo -sop I L Songe -soq I L Kanasi -sor I L Somrai -sos I L Seeku -sot sot sot st I L Southern Sotho -sou I L Southern Thai -sov I L Sonsorol -sow I L Sowanda -sox I L Swo -soy I L Miyobe -soz I L Temi -spa spa spa es I L Spanish -spb I L Sepa (Indonesia) -spc I L Sapé -spd I L Saep -spe I L Sepa (Papua New Guinea) -spg I L Sian -spi I L Saponi -spk I L Sengo -spl I L Selepet -spm I L Akukem -spo I L Spokane -spp I L Supyire Senoufo -spq I L Loreto-Ucayali Spanish -spr I L Saparua -sps I L Saposa -spt I L Spiti Bhoti -spu I L Sapuan -spv I L Sambalpuri -spx I A South Picene -spy I L Sabaot -sqa I L Shama-Sambuga -sqh I L Shau -sqi alb sqi sq M L Albanian -sqk I L Albanian Sign Language -sqm I L Suma -sqn I E Susquehannock -sqo I L Sorkhei -sqq I L Sou -sqr I H Siculo Arabic -sqs I L Sri Lankan Sign Language -sqt I L Soqotri -squ I L Squamish -sra I L Saruga -srb I L Sora -src I L Logudorese Sardinian -srd srd srd sc M L Sardinian -sre I L Sara -srf I L Nafi -srg I L Sulod -srh I L Sarikoli -sri I L Siriano -srk I L Serudung Murut -srl I L Isirawa -srm I L Saramaccan -srn srn srn I L Sranan Tongo -sro I L Campidanese Sardinian -srp srp srp sr I L Serbian -srq I L Sirionó -srr srr srr I L Serer -srs I L Sarsi -srt I L Sauri -sru I L Suruí -srv I L Southern Sorsoganon -srw I L Serua -srx I L Sirmauri -sry I L Sera -srz I L Shahmirzadi -ssb I L Southern Sama -ssc I L Suba-Simbiti -ssd I L Siroi -sse I L Balangingi -ssf I L Thao -ssg I L Seimat -ssh I L Shihhi Arabic -ssi I L Sansi -ssj I L Sausi -ssk I L Sunam -ssl I L Western Sisaala -ssm I L Semnam -ssn I L Waata -sso I L Sissano -ssp I L Spanish Sign Language -ssq I L So'a -ssr I L Swiss-French Sign Language -sss I L Sô -sst I L Sinasina -ssu I L Susuami -ssv I L Shark Bay -ssw ssw ssw ss I L Swati -ssx I L Samberigi -ssy I L Saho -ssz I L Sengseng -sta I L Settla -stb I L Northern Subanen -std I L Sentinel -ste I L Liana-Seti -stf I L Seta -stg I L Trieng -sth I L Shelta -sti I L Bulo Stieng -stj I L Matya Samo -stk I L Arammba -stl I L Stellingwerfs -stm I L Setaman -stn I L Owa -sto I L Stoney -stp I L Southeastern Tepehuan -stq I L Saterfriesisch -str I L Straits Salish -sts I L Shumashti -stt I L Budeh Stieng -stu I L Samtao -stv I L Silt'e -stw I L Satawalese -sty I L Siberian Tatar -sua I L Sulka -sub I L Suku -suc I L Western Subanon -sue I L Suena -sug I L Suganga -sui I L Suki -suj I L Shubi -suk suk suk I L Sukuma -sun sun sun su I L Sundanese -suq I L Suri -sur I L Mwaghavul -sus sus sus I L Susu -sut I E Subtiaba -suv I L Puroik -suw I L Sumbwa -sux sux sux I A Sumerian -suy I L Suyá -suz I L Sunwar -sva I L Svan -svb I L Ulau-Suain -svc I L Vincentian Creole English -sve I L Serili -svk I L Slovakian Sign Language -svm I L Slavomolisano -svr I L Savara -svs I L Savosavo -svx I E Skalvian -swa swa swa sw M L Swahili (macrolanguage) -swb I L Maore Comorian -swc I L Congo Swahili -swe swe swe sv I L Swedish -swf I L Sere -swg I L Swabian -swh I L Swahili (individual language) -swi I L Sui -swj I L Sira -swk I L Malawi Sena -swl I L Swedish Sign Language -swm I L Samosa -swn I L Sawknah -swo I L Shanenawa -swp I L Suau -swq I L Sharwa -swr I L Saweru -sws I L Seluwasan -swt I L Sawila -swu I L Suwawa -swv I L Shekhawati -sww I E Sowa -swx I L Suruahá -swy I L Sarua -sxb I L Suba -sxc I A Sicanian -sxe I L Sighu -sxg I L Shixing -sxk I E Southern Kalapuya -sxl I E Selian -sxm I L Samre -sxn I L Sangir -sxo I A Sorothaptic -sxr I L Saaroa -sxs I L Sasaru -sxu I L Upper Saxon -sxw I L Saxwe Gbe -sya I L Siang -syb I L Central Subanen -syc syc syc I H Classical Syriac -syi I L Seki -syk I L Sukur -syl I L Sylheti -sym I L Maya Samo -syn I L Senaya -syo I L Suoy -syr syr syr M L Syriac -sys I L Sinyar -syw I L Kagate -syy I L Al-Sayyid Bedouin Sign Language -sza I L Semelai -szb I L Ngalum -szc I L Semaq Beri -szd I E Seru -sze I L Seze -szg I L Sengele -szl I L Silesian -szn I L Sula -szp I L Suabo -szv I L Isu (Fako Division) -szw I L Sawai -taa I L Lower Tanana -tab I L Tabassaran -tac I L Lowland Tarahumara -tad I L Tause -tae I L Tariana -taf I L Tapirapé -tag I L Tagoi -tah tah tah ty I L Tahitian -taj I L Eastern Tamang -tak I L Tala -tal I L Tal -tam tam tam ta I L Tamil -tan I L Tangale -tao I L Yami -tap I L Taabwa -taq I L Tamasheq -tar I L Central Tarahumara -tas I E Tay Boi -tat tat tat tt I L Tatar -tau I L Upper Tanana -tav I L Tatuyo -taw I L Tai -tax I L Tamki -tay I L Atayal -taz I L Tocho -tba I L Aikanã -tbb I E Tapeba -tbc I L Takia -tbd I L Kaki Ae -tbe I L Tanimbili -tbf I L Mandara -tbg I L North Tairora -tbh I E Thurawal -tbi I L Gaam -tbj I L Tiang -tbk I L Calamian Tagbanwa -tbl I L Tboli -tbm I L Tagbu -tbn I L Barro Negro Tunebo -tbo I L Tawala -tbp I L Taworta -tbr I L Tumtum -tbs I L Tanguat -tbt I L Tembo (Kitembo) -tbu I E Tubar -tbv I L Tobo -tbw I L Tagbanwa -tbx I L Kapin -tby I L Tabaru -tbz I L Ditammari -tca I L Ticuna -tcb I L Tanacross -tcc I L Datooga -tcd I L Tafi -tce I L Southern Tutchone -tcf I L Malinaltepec Me'phaa -tcg I L Tamagario -tch I L Turks And Caicos Creole English -tci I L Wára -tck I L Tchitchege -tcl I E Taman (Myanmar) -tcm I L Tanahmerah -tcn I L Tichurong -tco I L Taungyo -tcp I L Tawr Chin -tcq I L Kaiy -tcs I L Torres Strait Creole -tct I L T'en -tcu I L Southeastern Tarahumara -tcw I L Tecpatlán Totonac -tcx I L Toda -tcy I L Tulu -tcz I L Thado Chin -tda I L Tagdal -tdb I L Panchpargania -tdc I L Emberá-Tadó -tdd I L Tai Nüa -tde I L Tiranige Diga Dogon -tdf I L Talieng -tdg I L Western Tamang -tdh I L Thulung -tdi I L Tomadino -tdj I L Tajio -tdk I L Tambas -tdl I L Sur -tdn I L Tondano -tdo I L Teme -tdq I L Tita -tdr I L Todrah -tds I L Doutai -tdt I L Tetun Dili -tdu I L Tempasuk Dusun -tdv I L Toro -tdx I L Tandroy-Mahafaly Malagasy -tdy I L Tadyawan -tea I L Temiar -teb I E Tetete -tec I L Terik -ted I L Tepo Krumen -tee I L Huehuetla Tepehua -tef I L Teressa -teg I L Teke-Tege -teh I L Tehuelche -tei I L Torricelli -tek I L Ibali Teke -tel tel tel te I L Telugu -tem tem tem I L Timne -ten I E Tama (Colombia) -teo I L Teso -tep I E Tepecano -teq I L Temein -ter ter ter I L Tereno -tes I L Tengger -tet tet tet I L Tetum -teu I L Soo -tev I L Teor -tew I L Tewa (USA) -tex I L Tennet -tey I L Tulishi -tfi I L Tofin Gbe -tfn I L Tanaina -tfo I L Tefaro -tfr I L Teribe -tft I L Ternate -tga I L Sagalla -tgb I L Tobilung -tgc I L Tigak -tgd I L Ciwogai -tge I L Eastern Gorkha Tamang -tgf I L Chalikha -tgh I L Tobagonian Creole English -tgi I L Lawunuia -tgj I L Tagin -tgk tgk tgk tg I L Tajik -tgl tgl tgl tl I L Tagalog -tgn I L Tandaganon -tgo I L Sudest -tgp I L Tangoa -tgq I L Tring -tgr I L Tareng -tgs I L Nume -tgt I L Central Tagbanwa -tgu I L Tanggu -tgv I E Tingui-Boto -tgw I L Tagwana Senoufo -tgx I L Tagish -tgy I E Togoyo -tgz I E Tagalaka -tha tha tha th I L Thai -thc I L Tai Hang Tong -thd I L Thayore -the I L Chitwania Tharu -thf I L Thangmi -thh I L Northern Tarahumara -thi I L Tai Long -thk I L Tharaka -thl I L Dangaura Tharu -thm I L Aheu -thn I L Thachanadan -thp I L Thompson -thq I L Kochila Tharu -thr I L Rana Tharu -ths I L Thakali -tht I L Tahltan -thu I L Thuri -thv I L Tahaggart Tamahaq -thw I L Thudam -thx I L The -thy I L Tha -thz I L Tayart Tamajeq -tia I L Tidikelt Tamazight -tic I L Tira -tid I L Tidong -tif I L Tifal -tig tig tig I L Tigre -tih I L Timugon Murut -tii I L Tiene -tij I L Tilung -tik I L Tikar -til I E Tillamook -tim I L Timbe -tin I L Tindi -tio I L Teop -tip I L Trimuris -tiq I L Tiéfo -tir tir tir ti I L Tigrinya -tis I L Masadiit Itneg -tit I L Tinigua -tiu I L Adasen -tiv tiv tiv I L Tiv -tiw I L Tiwi -tix I L Southern Tiwa -tiy I L Tiruray -tiz I L Tai Hongjin -tja I L Tajuasohn -tjg I L Tunjung -tji I L Northern Tujia -tjl I L Tai Laing -tjm I E Timucua -tjn I E Tonjon -tjo I L Temacine Tamazight -tjs I L Southern Tujia -tju I E Tjurruru -tjw I L Djabwurrung -tka I E Truká -tkb I L Buksa -tkd I L Tukudede -tke I L Takwane -tkf I E Tukumanféd -tkg I L Tesaka Malagasy -tkl tkl tkl I L Tokelau -tkm I E Takelma -tkn I L Toku-No-Shima -tkp I L Tikopia -tkq I L Tee -tkr I L Tsakhur -tks I L Takestani -tkt I L Kathoriya Tharu -tku I L Upper Necaxa Totonac -tkw I L Teanu -tkx I L Tangko -tkz I L Takua -tla I L Southwestern Tepehuan -tlb I L Tobelo -tlc I L Yecuatla Totonac -tld I L Talaud -tlf I L Telefol -tlg I L Tofanma -tlh tlh tlh I C Klingon -tli tli tli I L Tlingit -tlj I L Talinga-Bwisi -tlk I L Taloki -tll I L Tetela -tlm I L Tolomako -tln I L Talondo' -tlo I L Talodi -tlp I L Filomena Mata-Coahuitlán Totonac -tlq I L Tai Loi -tlr I L Talise -tls I L Tambotalo -tlt I L Teluti -tlu I L Tulehu -tlv I L Taliabu -tlx I L Khehek -tly I L Talysh -tma I L Tama (Chad) -tmb I L Katbol -tmc I L Tumak -tmd I L Haruai -tme I E Tremembé -tmf I L Toba-Maskoy -tmg I E Ternateño -tmh tmh tmh M L Tamashek -tmi I L Tutuba -tmj I L Samarokena -tmk I L Northwestern Tamang -tml I L Tamnim Citak -tmm I L Tai Thanh -tmn I L Taman (Indonesia) -tmo I L Temoq -tmp I L Tai Mène -tmq I L Tumleo -tmr I E Jewish Babylonian Aramaic (ca. 200-1200 CE) -tms I L Tima -tmt I L Tasmate -tmu I L Iau -tmv I L Tembo (Motembo) -tmw I L Temuan -tmy I L Tami -tmz I E Tamanaku -tna I L Tacana -tnb I L Western Tunebo -tnc I L Tanimuca-Retuarã -tnd I L Angosturas Tunebo -tne I L Tinoc Kallahan -tng I L Tobanga -tnh I L Maiani -tni I L Tandia -tnk I L Kwamera -tnl I L Lenakel -tnm I L Tabla -tnn I L North Tanna -tno I L Toromono -tnp I L Whitesands -tnq I E Taino -tnr I L Ménik -tns I L Tenis -tnt I L Tontemboan -tnu I L Tay Khang -tnv I L Tangchangya -tnw I L Tonsawang -tnx I L Tanema -tny I L Tongwe -tnz I L Tonga (Thailand) -tob I L Toba -toc I L Coyutla Totonac -tod I L Toma -toe I E Tomedes -tof I L Gizrra -tog tog tog I L Tonga (Nyasa) -toh I L Gitonga -toi I L Tonga (Zambia) -toj I L Tojolabal -tol I L Tolowa -tom I L Tombulu -ton ton ton to I L Tonga (Tonga Islands) -too I L Xicotepec De Juárez Totonac -top I L Papantla Totonac -toq I L Toposa -tor I L Togbo-Vara Banda -tos I L Highland Totonac -tou I L Tho -tov I L Upper Taromi -tow I L Jemez -tox I L Tobian -toy I L Topoiyo -toz I L To -tpa I L Taupota -tpc I L Azoyú Me'phaa -tpe I L Tippera -tpf I L Tarpia -tpg I L Kula -tpi tpi tpi I L Tok Pisin -tpj I L Tapieté -tpk I E Tupinikin -tpl I L Tlacoapa Me'phaa -tpm I L Tampulma -tpn I E Tupinambá -tpo I L Tai Pao -tpp I L Pisaflores Tepehua -tpq I L Tukpa -tpr I L Tuparí -tpt I L Tlachichilco Tepehua -tpu I L Tampuan -tpv I L Tanapag -tpw I E Tupí -tpx I L Acatepec Me'phaa -tpy I L Trumai -tpz I L Tinputz -tqb I L Tembé -tql I L Lehali -tqm I L Turumsa -tqn I L Tenino -tqo I L Toaripi -tqp I L Tomoip -tqq I L Tunni -tqr I E Torona -tqt I L Western Totonac -tqu I L Touo -tqw I E Tonkawa -tra I L Tirahi -trb I L Terebu -trc I L Copala Triqui -trd I L Turi -tre I L East Tarangan -trf I L Trinidadian Creole English -trg I L Lishán Didán -trh I L Turaka -tri I L Trió -trj I L Toram -trl I L Traveller Scottish -trm I L Tregami -trn I L Trinitario -tro I L Tarao Naga -trp I L Kok Borok -trq I L San Martín Itunyoso Triqui -trr I L Taushiro -trs I L Chicahuaxtla Triqui -trt I L Tunggare -tru I L Turoyo -trv I L Taroko -trw I L Torwali -trx I L Tringgus-Sembaan Bidayuh -try I E Turung -trz I E Torá -tsa I L Tsaangi -tsb I L Tsamai -tsc I L Tswa -tsd I L Tsakonian -tse I L Tunisian Sign Language -tsf I L Southwestern Tamang -tsg I L Tausug -tsh I L Tsuvan -tsi tsi tsi I L Tsimshian -tsj I L Tshangla -tsk I L Tseku -tsl I L Ts'ün-Lao -tsm I L Turkish Sign Language -tsn tsn tsn tn I L Tswana -tso tso tso ts I L Tsonga -tsp I L Northern Toussian -tsq I L Thai Sign Language -tsr I L Akei -tss I L Taiwan Sign Language -tst I L Tondi Songway Kiini -tsu I L Tsou -tsv I L Tsogo -tsw I L Tsishingini -tsx I L Mubami -tsy I L Tebul Sign Language -tsz I L Purepecha -tta I E Tutelo -ttb I L Gaa -ttc I L Tektiteko -ttd I L Tauade -tte I L Bwanabwana -ttf I L Tuotomb -ttg I L Tutong -tth I L Upper Ta'oih -tti I L Tobati -ttj I L Tooro -ttk I L Totoro -ttl I L Totela -ttm I L Northern Tutchone -ttn I L Towei -tto I L Lower Ta'oih -ttp I L Tombelala -ttq I L Tawallammat Tamajaq -ttr I L Tera -tts I L Northeastern Thai -ttt I L Muslim Tat -ttu I L Torau -ttv I L Titan -ttw I L Long Wat -tty I L Sikaritai -ttz I L Tsum -tua I L Wiarumus -tub I L Tübatulabal -tuc I L Mutu -tud I E Tuxá -tue I L Tuyuca -tuf I L Central Tunebo -tug I L Tunia -tuh I L Taulil -tui I L Tupuri -tuj I L Tugutil -tuk tuk tuk tk I L Turkmen -tul I L Tula -tum tum tum I L Tumbuka -tun I E Tunica -tuo I L Tucano -tuq I L Tedaga -tur tur tur tr I L Turkish -tus I L Tuscarora -tuu I L Tututni -tuv I L Turkana -tux I E Tuxináwa -tuy I L Tugen -tuz I L Turka -tva I L Vaghua -tvd I L Tsuvadi -tve I L Te'un -tvk I L Southeast Ambrym -tvl tvl tvl I L Tuvalu -tvm I L Tela-Masbuar -tvn I L Tavoyan -tvo I L Tidore -tvs I L Taveta -tvt I L Tutsa Naga -tvu I L Tunen -tvw I L Sedoa -tvy I E Timor Pidgin -twa I E Twana -twb I L Western Tawbuid -twc I E Teshenawa -twd I L Twents -twe I L Tewa (Indonesia) -twf I L Northern Tiwa -twg I L Tereweng -twh I L Tai Dón -twi twi twi tw I L Twi -twl I L Tawara -twm I L Tawang Monpa -twn I L Twendi -two I L Tswapong -twp I L Ere -twq I L Tasawaq -twr I L Southwestern Tarahumara -twt I E Turiwára -twu I L Termanu -tww I L Tuwari -twx I L Tewe -twy I L Tawoyan -txa I L Tombonuo -txb I A Tokharian B -txc I E Tsetsaut -txe I L Totoli -txg I A Tangut -txh I A Thracian -txi I L Ikpeng -txm I L Tomini -txn I L West Tarangan -txo I L Toto -txq I L Tii -txr I A Tartessian -txs I L Tonsea -txt I L Citak -txu I L Kayapó -txx I L Tatana -txy I L Tanosy Malagasy -tya I L Tauya -tye I L Kyanga -tyh I L O'du -tyi I L Teke-Tsaayi -tyj I L Tai Do -tyl I L Thu Lao -tyn I L Kombai -typ I E Thaypan -tyr I L Tai Daeng -tys I L Tày Sa Pa -tyt I L Tày Tac -tyu I L Kua -tyv tyv tyv I L Tuvinian -tyx I L Teke-Tyee -tyz I L Tày -tza I L Tanzanian Sign Language -tzh I L Tzeltal -tzj I L Tz'utujil -tzl I C Talossan -tzm I L Central Atlas Tamazight -tzn I L Tugun -tzo I L Tzotzil -tzx I L Tabriak -uam I E Uamué -uan I L Kuan -uar I L Tairuma -uba I L Ubang -ubi I L Ubi -ubl I L Buhi'non Bikol -ubr I L Ubir -ubu I L Umbu-Ungu -uby I E Ubykh -uda I L Uda -ude I L Udihe -udg I L Muduga -udi I L Udi -udj I L Ujir -udl I L Wuzlam -udm udm udm I L Udmurt -udu I L Uduk -ues I L Kioko -ufi I L Ufim -uga uga uga I A Ugaritic -ugb I E Kuku-Ugbanh -uge I L Ughele -ugn I L Ugandan Sign Language -ugo I L Ugong -ugy I L Uruguayan Sign Language -uha I L Uhami -uhn I L Damal -uig uig uig ug I L Uighur -uis I L Uisai -uiv I L Iyive -uji I L Tanjijili -uka I L Kaburi -ukg I L Ukuriguma -ukh I L Ukhwejo -ukl I L Ukrainian Sign Language -ukp I L Ukpe-Bayobiri -ukq I L Ukwa -ukr ukr ukr uk I L Ukrainian -uks I L Urubú-Kaapor Sign Language -uku I L Ukue -ukw I L Ukwuani-Aboh-Ndoni -uky I E Kuuk-Yak -ula I L Fungwa -ulb I L Ulukwumi -ulc I L Ulch -ule I E Lule -ulf I L Usku -uli I L Ulithian -ulk I L Meriam -ull I L Ullatan -ulm I L Ulumanda' -uln I L Unserdeutsch -ulu I L Uma' Lung -ulw I L Ulwa -uma I L Umatilla -umb umb umb I L Umbundu -umc I A Marrucinian -umd I E Umbindhamu -umg I E Umbuygamu -umi I L Ukit -umm I L Umon -umn I L Makyan Naga -umo I E Umotína -ump I L Umpila -umr I E Umbugarla -ums I L Pendau -umu I L Munsee -una I L North Watut -und und und S S Undetermined -une I L Uneme -ung I L Ngarinyin -unk I L Enawené-Nawé -unm I E Unami -unn I L Kurnai -unr I L Mundari -unu I L Unubahe -unx I L Munda -unz I L Unde Kaili -uok I L Uokha -upi I L Umeda -upv I L Uripiv-Wala-Rano-Atchin -ura I L Urarina -urb I L Urubú-Kaapor -urc I E Urningangg -urd urd urd ur I L Urdu -ure I L Uru -urf I E Uradhi -urg I L Urigina -urh I L Urhobo -uri I L Urim -urk I L Urak Lawoi' -url I L Urali -urm I L Urapmin -urn I L Uruangnirin -uro I L Ura (Papua New Guinea) -urp I L Uru-Pa-In -urr I L Lehalurup -urt I L Urat -uru I E Urumi -urv I E Uruava -urw I L Sop -urx I L Urimo -ury I L Orya -urz I L Uru-Eu-Wau-Wau -usa I L Usarufa -ush I L Ushojo -usi I L Usui -usk I L Usaghade -usp I L Uspanteco -usu I L Uya -uta I L Otank -ute I L Ute-Southern Paiute -utp I L Amba (Solomon Islands) -utr I L Etulo -utu I L Utu -uum I L Urum -uun I L Kulon-Pazeh -uur I L Ura (Vanuatu) -uuu I L U -uve I L West Uvean -uvh I L Uri -uvl I L Lote -uwa I L Kuku-Uwanh -uya I L Doko-Uyanga -uzb uzb uzb uz M L Uzbek -uzn I L Northern Uzbek -uzs I L Southern Uzbek -vaa I L Vaagri Booli -vae I L Vale -vaf I L Vafsi -vag I L Vagla -vah I L Varhadi-Nagpuri -vai vai vai I L Vai -vaj I L Vasekela Bushman -val I L Vehes -vam I L Vanimo -van I L Valman -vao I L Vao -vap I L Vaiphei -var I L Huarijio -vas I L Vasavi -vau I L Vanuma -vav I L Varli -vay I L Wayu -vbb I L Southeast Babar -vbk I L Southwestern Bontok -vec I L Venetian -ved I L Veddah -vel I L Veluws -vem I L Vemgo-Mabas -ven ven ven ve I L Venda -veo I E Ventureño -vep I L Veps -ver I L Mom Jango -vgr I L Vaghri -vgt I L Vlaamse Gebarentaal -vic I L Virgin Islands Creole English -vid I L Vidunda -vie vie vie vi I L Vietnamese -vif I L Vili -vig I L Viemo -vil I L Vilela -vin I L Vinza -vis I L Vishavan -vit I L Viti -viv I L Iduna -vka I E Kariyarra -vki I L Ija-Zuba -vkj I L Kujarge -vkk I L Kaur -vkl I L Kulisusu -vkm I E Kamakan -vko I L Kodeoha -vkp I L Korlai Creole Portuguese -vkt I L Tenggarong Kutai Malay -vku I L Kurrama -vlp I L Valpei -vls I L Vlaams -vma I L Martuyhunira -vmb I E Barbaram -vmc I L Juxtlahuaca Mixtec -vmd I L Mudu Koraga -vme I L East Masela -vmf I L Mainfränkisch -vmg I L Lungalunga -vmh I L Maraghei -vmi I E Miwa -vmj I L Ixtayutla Mixtec -vmk I L Makhuwa-Shirima -vml I E Malgana -vmm I L Mitlatongo Mixtec -vmp I L Soyaltepec Mazatec -vmq I L Soyaltepec Mixtec -vmr I L Marenje -vms I E Moksela -vmu I E Muluridyi -vmv I E Valley Maidu -vmw I L Makhuwa -vmx I L Tamazola Mixtec -vmy I L Ayautla Mazatec -vmz I L Mazatlán Mazatec -vnk I L Vano -vnm I L Vinmavis -vnp I L Vunapu -vol vol vol vo I C Volapük -vor I L Voro -vot vot vot I L Votic -vra I L Vera'a -vro I L Võro -vrs I L Varisi -vrt I L Burmbar -vsi I L Moldova Sign Language -vsl I L Venezuelan Sign Language -vsv I L Valencian Sign Language -vto I L Vitou -vum I L Vumbu -vun I L Vunjo -vut I L Vute -vwa I L Awa (China) -waa I L Walla Walla -wab I L Wab -wac I L Wasco-Wishram -wad I L Wandamen -wae I L Walser -waf I E Wakoná -wag I L Wa'ema -wah I L Watubela -wai I L Wares -waj I L Waffa -wal wal wal I L Wolaytta -wam I E Wampanoag -wan I L Wan -wao I E Wappo -wap I L Wapishana -waq I L Wageman -war war war I L Waray (Philippines) -was was was I L Washo -wat I L Kaninuwa -wau I L Waurá -wav I L Waka -waw I L Waiwai -wax I L Watam -way I L Wayana -waz I L Wampur -wba I L Warao -wbb I L Wabo -wbe I L Waritai -wbf I L Wara -wbh I L Wanda -wbi I L Vwanji -wbj I L Alagwa -wbk I L Waigali -wbl I L Wakhi -wbm I L Wa -wbp I L Warlpiri -wbq I L Waddar -wbr I L Wagdi -wbt I L Wanman -wbv I L Wajarri -wbw I L Woi -wca I L Yanomámi -wci I L Waci Gbe -wdd I L Wandji -wdg I L Wadaginam -wdj I L Wadjiginy -wdk I E Wadikali -wdu I E Wadjigu -wdy I E Wadjabangayi -wea I E Wewaw -wec I L Wè Western -wed I L Wedau -weg I L Wergaia -weh I L Weh -wei I L Kiunum -wem I L Weme Gbe -weo I L Wemale -wep I L Westphalien -wer I L Weri -wes I L Cameroon Pidgin -wet I L Perai -weu I L Rawngtu Chin -wew I L Wejewa -wfg I L Yafi -wga I E Wagaya -wgb I L Wagawaga -wgg I E Wangganguru -wgi I L Wahgi -wgo I L Waigeo -wgu I E Wirangu -wgy I L Warrgamay -wha I L Manusela -whg I L North Wahgi -whk I L Wahau Kenyah -whu I L Wahau Kayan -wib I L Southern Toussian -wic I L Wichita -wie I E Wik-Epa -wif I E Wik-Keyangan -wig I L Wik-Ngathana -wih I L Wik-Me'anha -wii I L Minidien -wij I L Wik-Iiyanh -wik I L Wikalkan -wil I E Wilawila -wim I L Wik-Mungkan -win I L Ho-Chunk -wir I E Wiraféd -wiu I L Wiru -wiv I L Vitu -wiy I E Wiyot -wja I L Waja -wji I L Warji -wka I E Kw'adza -wkb I L Kumbaran -wkd I L Wakde -wkl I L Kalanadi -wku I L Kunduvadi -wkw I E Wakawaka -wky I E Wangkayutyuru -wla I L Walio -wlc I L Mwali Comorian -wle I L Wolane -wlg I L Kunbarlang -wli I L Waioli -wlk I E Wailaki -wll I L Wali (Sudan) -wlm I H Middle Welsh -wln wln wln wa I L Walloon -wlo I L Wolio -wlr I L Wailapa -wls I L Wallisian -wlu I E Wuliwuli -wlv I L Wichí Lhamtés Vejoz -wlw I L Walak -wlx I L Wali (Ghana) -wly I E Waling -wma I E Mawa (Nigeria) -wmb I L Wambaya -wmc I L Wamas -wmd I L Mamaindé -wme I L Wambule -wmh I L Waima'a -wmi I E Wamin -wmm I L Maiwa (Indonesia) -wmn I E Waamwang -wmo I L Wom (Papua New Guinea) -wms I L Wambon -wmt I L Walmajarri -wmw I L Mwani -wmx I L Womo -wnb I L Wanambre -wnc I L Wantoat -wnd I E Wandarang -wne I L Waneci -wng I L Wanggom -wni I L Ndzwani Comorian -wnk I L Wanukaka -wnm I E Wanggamala -wnn I E Wunumara -wno I L Wano -wnp I L Wanap -wnu I L Usan -wnw I L Wintu -wny I L Wanyi -woa I L Tyaraity -wob I L Wè Northern -woc I L Wogeo -wod I L Wolani -woe I L Woleaian -wof I L Gambian Wolof -wog I L Wogamusin -woi I L Kamang -wok I L Longto -wol wol wol wo I L Wolof -wom I L Wom (Nigeria) -won I L Wongo -woo I L Manombai -wor I L Woria -wos I L Hanga Hundi -wow I L Wawonii -woy I E Weyto -wpc I L Maco -wra I L Warapu -wrb I E Warluwara -wrd I L Warduji -wrg I E Warungu -wrh I E Wiradhuri -wri I E Wariyangga -wrk I L Garrwa -wrl I L Warlmanpa -wrm I L Warumungu -wrn I L Warnang -wro I E Worrorra -wrp I L Waropen -wrr I L Wardaman -wrs I L Waris -wru I L Waru -wrv I L Waruna -wrw I E Gugu Warra -wrx I L Wae Rana -wry I L Merwari -wrz I E Waray (Australia) -wsa I L Warembori -wsi I L Wusi -wsk I L Waskia -wsr I L Owenia -wss I L Wasa -wsu I E Wasu -wsv I E Wotapuri-Katarqalai -wtf I L Watiwa -wth I E Wathawurrung -wti I L Berta -wtk I L Watakataui -wtm I L Mewati -wtw I L Wotu -wua I L Wikngenchera -wub I L Wunambal -wud I L Wudu -wuh I L Wutunhua -wul I L Silimo -wum I L Wumbvu -wun I L Bungu -wur I E Wurrugu -wut I L Wutung -wuu I L Wu Chinese -wuv I L Wuvulu-Aua -wux I L Wulna -wuy I L Wauyai -wwa I L Waama -wwb I E Wakabunga -wwo I L Wetamut -wwr I E Warrwa -www I L Wawa -wxa I L Waxianghua -wxw I E Wardandi -wya I L Wyandot -wyb I L Wangaaybuwan-Ngiyambaa -wyi I E Woiwurrung -wym I L Wymysorys -wyr I L Wayoró -wyy I L Western Fijian -xaa I H Andalusian Arabic -xab I L Sambe -xac I L Kachari -xad I E Adai -xae I A Aequian -xag I E Aghwan -xai I E Kaimbé -xal xal xal I L Kalmyk -xam I E /Xam -xan I L Xamtanga -xao I L Khao -xap I E Apalachee -xaq I A Aquitanian -xar I E Karami -xas I E Kamas -xat I L Katawixi -xau I L Kauwera -xav I L Xavánte -xaw I L Kawaiisu -xay I L Kayan Mahakam -xba I E Kamba (Brazil) -xbb I E Lower Burdekin -xbc I A Bactrian -xbd I E Bindal -xbe I E Bigambal -xbg I E Bunganditj -xbi I L Kombio -xbj I E Birrpayi -xbm I H Middle Breton -xbn I E Kenaboi -xbo I E Bolgarian -xbp I E Bibbulman -xbr I L Kambera -xbw I E Kambiwá -xbx I E Kabixí -xby I L Batyala -xcb I E Cumbric -xcc I A Camunic -xce I A Celtiberian -xcg I A Cisalpine Gaulish -xch I E Chemakum -xcl I H Classical Armenian -xcm I E Comecrudo -xcn I E Cotoname -xco I A Chorasmian -xcr I A Carian -xct I H Classical Tibetan -xcu I E Curonian -xcv I E Chuvantsy -xcw I E Coahuilteco -xcy I E Cayuse -xda I L Darkinyung -xdc I A Dacian -xdk I E Dharuk -xdm I A Edomite -xdy I L Malayic Dayak -xeb I A Eblan -xed I L Hdi -xeg I E //Xegwi -xel I L Kelo -xem I L Kembayan -xep I A Epi-Olmec -xer I L Xerénte -xes I L Kesawai -xet I L Xetá -xeu I L Keoru-Ahia -xfa I A Faliscan -xga I A Galatian -xgb I E Gbin -xgd I E Gudang -xgf I E Gabrielino-Fernandeño -xgg I E Goreng -xgi I E Garingbal -xgl I E Galindan -xgm I E Guwinmal -xgr I E Garza -xgu I L Unggumi -xgw I E Guwa -xha I A Harami -xhc I E Hunnic -xhd I A Hadrami -xhe I L Khetrani -xho xho xho xh I L Xhosa -xhr I A Hernican -xht I A Hattic -xhu I A Hurrian -xhv I L Khua -xib I A Iberian -xii I L Xiri -xil I A Illyrian -xin I E Xinca -xip I E Xipináwa -xir I E Xiriâna -xiv I A Indus Valley Language -xiy I L Xipaya -xjb I E Minjungbal -xjt I E Jaitmatang -xka I L Kalkoti -xkb I L Northern Nago -xkc I L Kho'ini -xkd I L Mendalam Kayan -xke I L Kereho -xkf I L Khengkha -xkg I L Kagoro -xkh I L Karahawyana -xki I L Kenyan Sign Language -xkj I L Kajali -xkk I L Kaco' -xkl I L Mainstream Kenyah -xkn I L Kayan River Kayan -xko I L Kiorr -xkp I L Kabatei -xkq I L Koroni -xkr I E Xakriabá -xks I L Kumbewaha -xkt I L Kantosi -xku I L Kaamba -xkv I L Kgalagadi -xkw I L Kembra -xkx I L Karore -xky I L Uma' Lasan -xkz I L Kurtokha -xla I L Kamula -xlb I E Loup B -xlc I A Lycian -xld I A Lydian -xle I A Lemnian -xlg I A Ligurian (Ancient) -xli I A Liburnian -xln I A Alanic -xlo I E Loup A -xlp I A Lepontic -xls I A Lusitanian -xlu I A Cuneiform Luwian -xly I A Elymian -xma I L Mushungulu -xmb I L Mbonga -xmc I L Makhuwa-Marrevone -xmd I L Mbudum -xme I A Median -xmf I L Mingrelian -xmg I L Mengaka -xmh I L Kuku-Muminh -xmj I L Majera -xmk I A Ancient Macedonian -xml I L Malaysian Sign Language -xmm I L Manado Malay -xmn I H Manichaean Middle Persian -xmo I L Morerebi -xmp I E Kuku-Mu'inh -xmq I E Kuku-Mangk -xmr I A Meroitic -xms I L Moroccan Sign Language -xmt I L Matbat -xmu I E Kamu -xmv I L Antankarana Malagasy -xmw I L Tsimihety Malagasy -xmx I L Maden -xmy I L Mayaguduna -xmz I L Mori Bawah -xna I A Ancient North Arabian -xnb I L Kanakanabu -xng I H Middle Mongolian -xnh I L Kuanhua -xni I E Ngarigu -xnk I E Nganakarti -xnn I L Northern Kankanay -xno I H Anglo-Norman -xnr I L Kangri -xns I L Kanashi -xnt I E Narragansett -xnu I E Nukunul -xny I L Nyiyaparli -xnz I L Kenzi -xoc I E O'chi'chi' -xod I L Kokoda -xog I L Soga -xoi I L Kominimung -xok I L Xokleng -xom I L Komo (Sudan) -xon I L Konkomba -xoo I E Xukurú -xop I L Kopar -xor I L Korubo -xow I L Kowaki -xpa I E Pirriya -xpc I E Pecheneg -xpe I L Liberia Kpelle -xpg I A Phrygian -xpi I E Pictish -xpj I E Mpalitjanh -xpk I L Kulina Pano -xpm I E Pumpokol -xpn I E Kapinawá -xpo I E Pochutec -xpp I E Puyo-Paekche -xpq I E Mohegan-Pequot -xpr I A Parthian -xps I E Pisidian -xpt I E Punthamara -xpu I A Punic -xpy I E Puyo -xqa I H Karakhanid -xqt I A Qatabanian -xra I L Krahô -xrb I L Eastern Karaboro -xrd I E Gundungurra -xre I L Kreye -xrg I E Minang -xri I L Krikati-Timbira -xrm I E Armazic -xrn I E Arin -xrq I E Karranga -xrr I A Raetic -xrt I E Aranama-Tamique -xru I L Marriammu -xrw I L Karawa -xsa I A Sabaean -xsb I L Sambal -xsc I A Scythian -xsd I A Sidetic -xse I L Sempan -xsh I L Shamang -xsi I L Sio -xsj I L Subi -xsl I L South Slavey -xsm I L Kasem -xsn I L Sanga (Nigeria) -xso I E Solano -xsp I L Silopi -xsq I L Makhuwa-Saka -xsr I L Sherpa -xss I E Assan -xsu I L Sanumá -xsv I E Sudovian -xsy I L Saisiyat -xta I L Alcozauca Mixtec -xtb I L Chazumba Mixtec -xtc I L Katcha-Kadugli-Miri -xtd I L Diuxi-Tilantongo Mixtec -xte I L Ketengban -xtg I A Transalpine Gaulish -xth I E Yitha Yitha -xti I L Sinicahua Mixtec -xtj I L San Juan Teita Mixtec -xtl I L Tijaltepec Mixtec -xtm I L Magdalena Peñasco Mixtec -xtn I L Northern Tlaxiaco Mixtec -xto I A Tokharian A -xtp I L San Miguel Piedras Mixtec -xtq I H Tumshuqese -xtr I A Early Tripuri -xts I L Sindihui Mixtec -xtt I L Tacahua Mixtec -xtu I L Cuyamecalco Mixtec -xtv I E Thawa -xtw I L Tawandê -xty I L Yoloxochitl Mixtec -xtz I E Tasmanian -xua I L Alu Kurumba -xub I L Betta Kurumba -xud I E Umiida -xug I L Kunigami -xuj I L Jennu Kurumba -xul I E Ngunawal -xum I A Umbrian -xun I E Unggaranggu -xuo I L Kuo -xup I E Upper Umpqua -xur I A Urartian -xut I E Kuthant -xuu I L Kxoe -xve I A Venetic -xvi I L Kamviri -xvn I A Vandalic -xvo I A Volscian -xvs I A Vestinian -xwa I L Kwaza -xwc I E Woccon -xwd I E Wadi Wadi -xwe I L Xwela Gbe -xwg I L Kwegu -xwj I E Wajuk -xwk I E Wangkumara -xwl I L Western Xwla Gbe -xwo I E Written Oirat -xwr I L Kwerba Mamberamo -xwt I E Wotjobaluk -xww I E Wemba Wemba -xxb I E Boro (Ghana) -xxk I L Ke'o -xxm I E Minkin -xxr I E Koropó -xxt I E Tambora -xya I E Yaygir -xyb I E Yandjibara -xyj I E Mayi-Yapi -xyk I E Mayi-Kulan -xyl I E Yalakalore -xyt I E Mayi-Thakurti -xyy I L Yorta Yorta -xzh I A Zhang-Zhung -xzm I E Zemgalian -xzp I H Ancient Zapotec -yaa I L Yaminahua -yab I L Yuhup -yac I L Pass Valley Yali -yad I L Yagua -yae I L Pumé -yaf I L Yaka (Democratic Republic of Congo) -yag I L Yámana -yah I L Yazgulyam -yai I L Yagnobi -yaj I L Banda-Yangere -yak I L Yakama -yal I L Yalunka -yam I L Yamba -yan I L Mayangna -yao yao yao I L Yao -yap yap yap I L Yapese -yaq I L Yaqui -yar I L Yabarana -yas I L Nugunu (Cameroon) -yat I L Yambeta -yau I L Yuwana -yav I L Yangben -yaw I L Yawalapití -yax I L Yauma -yay I L Agwagwune -yaz I L Lokaa -yba I L Yala -ybb I L Yemba -ybe I L West Yugur -ybh I L Yakha -ybi I L Yamphu -ybj I L Hasha -ybk I L Bokha -ybl I L Yukuben -ybm I L Yaben -ybn I E Yabaâna -ybo I L Yabong -ybx I L Yawiyo -yby I L Yaweyuha -ych I L Chesu -ycl I L Lolopo -ycn I L Yucuna -ycp I L Chepya -yda I E Yanda -ydd I L Eastern Yiddish -yde I L Yangum Dey -ydg I L Yidgha -ydk I L Yoidik -yds I L Yiddish Sign Language -yea I L Ravula -yec I L Yeniche -yee I L Yimas -yei I E Yeni -yej I L Yevanic -yel I L Yela -yer I L Tarok -yes I L Nyankpa -yet I L Yetfa -yeu I L Yerukula -yev I L Yapunda -yey I L Yeyi -yga I E Malyangapa -ygi I E Yiningayi -ygl I L Yangum Gel -ygm I L Yagomi -ygp I L Gepo -ygr I L Yagaria -ygu I L Yugul -ygw I L Yagwoia -yha I L Baha Buyang -yhd I L Judeo-Iraqi Arabic -yhl I L Hlepho Phowa -yia I L Yinggarda -yid yid yid yi M L Yiddish -yif I L Ache -yig I L Wusa Nasu -yih I L Western Yiddish -yii I L Yidiny -yij I L Yindjibarndi -yik I L Dongshanba Lalo -yil I E Yindjilandji -yim I L Yimchungru Naga -yin I L Yinchia -yip I L Pholo -yiq I L Miqie -yir I L North Awyu -yis I L Yis -yit I L Eastern Lalu -yiu I L Awu -yiv I L Northern Nisu -yix I L Axi Yi -yiz I L Azhe -yka I L Yakan -ykg I L Northern Yukaghir -yki I L Yoke -ykk I L Yakaikeke -ykl I L Khlula -ykm I L Kap -ykn I L Kua-nsi -yko I L Yasa -ykr I L Yekora -ykt I L Kathu -yku I L Kuamasi -yky I L Yakoma -yla I L Yaul -ylb I L Yaleba -yle I L Yele -ylg I L Yelogu -yli I L Angguruk Yali -yll I L Yil -ylm I L Limi -yln I L Langnian Buyang -ylo I L Naluo Yi -ylr I E Yalarnnga -ylu I L Aribwaung -yly I L Nyâlayu -ymb I L Yambes -ymc I L Southern Muji -ymd I L Muda -yme I E Yameo -ymg I L Yamongeri -ymh I L Mili -ymi I L Moji -ymk I L Makwe -yml I L Iamalele -ymm I L Maay -ymn I L Yamna -ymo I L Yangum Mon -ymp I L Yamap -ymq I L Qila Muji -ymr I L Malasar -yms I A Mysian -ymt I E Mator-Taygi-Karagas -ymx I L Northern Muji -ymz I L Muzi -yna I L Aluo -ynd I E Yandruwandha -yne I L Lang'e -yng I L Yango -ynh I L Yangho -ynk I L Naukan Yupik -ynl I L Yangulam -ynn I E Yana -yno I L Yong -ynq I L Yendang -yns I L Yansi -ynu I E Yahuna -yob I E Yoba -yog I L Yogad -yoi I L Yonaguni -yok I L Yokuts -yol I E Yola -yom I L Yombe -yon I L Yongkom -yor yor yor yo I L Yoruba -yot I L Yotti -yox I L Yoron -yoy I L Yoy -ypa I L Phala -ypb I L Labo Phowa -ypg I L Phola -yph I L Phupha -ypm I L Phuma -ypn I L Ani Phowa -ypo I L Alo Phola -ypp I L Phupa -ypz I L Phuza -yra I L Yerakai -yrb I L Yareba -yre I L Yaouré -yri I L Yarí -yrk I L Nenets -yrl I L Nhengatu -yrm I L Yirrk-Mel -yrn I L Yerong -yrs I L Yarsun -yrw I L Yarawata -yry I L Yarluyandi -ysc I E Yassic -ysd I L Samatao -ysg I L Sonaga -ysl I L Yugoslavian Sign Language -ysn I L Sani -yso I L Nisi (China) -ysp I L Southern Lolopo -ysr I E Sirenik Yupik -yss I L Yessan-Mayo -ysy I L Sanie -yta I L Talu -ytl I L Tanglang -ytp I L Thopho -ytw I L Yout Wam -yty I E Yatay -yua I L Yucateco -yub I E Yugambal -yuc I L Yuchi -yud I L Judeo-Tripolitanian Arabic -yue I L Yue Chinese -yuf I L Havasupai-Walapai-Yavapai -yug I E Yug -yui I L Yurutí -yuj I L Karkar-Yuri -yuk I E Yuki -yul I L Yulu -yum I L Quechan -yun I L Bena (Nigeria) -yup I L Yukpa -yuq I L Yuqui -yur I L Yurok -yut I L Yopno -yuu I L Yugh -yuw I L Yau (Morobe Province) -yux I L Southern Yukaghir -yuy I L East Yugur -yuz I L Yuracare -yva I L Yawa -yvt I E Yavitero -ywa I L Kalou -ywg I L Yinhawangka -ywl I L Western Lalu -ywn I L Yawanawa -ywq I L Wuding-Luquan Yi -ywr I L Yawuru -ywt I L Xishanba Lalo -ywu I L Wumeng Nasu -yww I E Yawarawarga -yxa I E Mayawali -yxg I E Yagara -yxl I E Yardliyawarra -yxm I E Yinwum -yxu I E Yuyu -yxy I E Yabula Yabula -yyr I E Yir Yoront -yyu I L Yau (Sandaun Province) -yyz I L Ayizi -yzg I L E'ma Buyang -yzk I L Zokhuo -zaa I L Sierra de Juárez Zapotec -zab I L San Juan Guelavía Zapotec -zac I L Ocotlán Zapotec -zad I L Cajonos Zapotec -zae I L Yareni Zapotec -zaf I L Ayoquesco Zapotec -zag I L Zaghawa -zah I L Zangwal -zai I L Isthmus Zapotec -zaj I L Zaramo -zak I L Zanaki -zal I L Zauzou -zam I L Miahuatlán Zapotec -zao I L Ozolotepec Zapotec -zap zap zap M L Zapotec -zaq I L Aloápam Zapotec -zar I L Rincón Zapotec -zas I L Santo Domingo Albarradas Zapotec -zat I L Tabaa Zapotec -zau I L Zangskari -zav I L Yatzachi Zapotec -zaw I L Mitla Zapotec -zax I L Xadani Zapotec -zay I L Zayse-Zergulla -zaz I L Zari -zbc I L Central Berawan -zbe I L East Berawan -zbl zbl zbl I C Blissymbols -zbt I L Batui -zbw I L West Berawan -zca I L Coatecas Altas Zapotec -zch I L Central Hongshuihe Zhuang -zdj I L Ngazidja Comorian -zea I L Zeeuws -zeg I L Zenag -zeh I L Eastern Hongshuihe Zhuang -zen zen zen I L Zenaga -zga I L Kinga -zgb I L Guibei Zhuang -zgh I L Standard Moroccan Tamazight -zgm I L Minz Zhuang -zgn I L Guibian Zhuang -zgr I L Magori -zha zha zha za M L Zhuang -zhb I L Zhaba -zhd I L Dai Zhuang -zhi I L Zhire -zhn I L Nong Zhuang -zho chi zho zh M L Chinese -zhw I L Zhoa -zia I L Zia -zib I L Zimbabwe Sign Language -zik I L Zimakani -zil I L Zialo -zim I L Mesme -zin I L Zinza -zir I E Ziriya -ziw I L Zigula -ziz I L Zizilivakan -zka I L Kaimbulawa -zkb I E Koibal -zkd I L Kadu -zkg I E Koguryo -zkh I E Khorezmian -zkk I E Karankawa -zkn I L Kanan -zko I E Kott -zkp I E São Paulo Kaingáng -zkr I L Zakhring -zkt I E Kitan -zku I E Kaurna -zkv I E Krevinian -zkz I E Khazar -zlj I L Liujiang Zhuang -zlm I L Malay (individual language) -zln I L Lianshan Zhuang -zlq I L Liuqian Zhuang -zma I L Manda (Australia) -zmb I L Zimba -zmc I E Margany -zmd I L Maridan -zme I E Mangerr -zmf I L Mfinu -zmg I L Marti Ke -zmh I E Makolkol -zmi I L Negeri Sembilan Malay -zmj I L Maridjabin -zmk I E Mandandanyi -zml I L Madngele -zmm I L Marimanindji -zmn I L Mbangwe -zmo I L Molo -zmp I L Mpuono -zmq I L Mituku -zmr I L Maranunggu -zms I L Mbesa -zmt I L Maringarr -zmu I E Muruwari -zmv I E Mbariman-Gudhinma -zmw I L Mbo (Democratic Republic of Congo) -zmx I L Bomitaba -zmy I L Mariyedi -zmz I L Mbandja -zna I L Zan Gula -zne I L Zande (individual language) -zng I L Mang -znk I E Manangkari -zns I L Mangas -zoc I L Copainalá Zoque -zoh I L Chimalapa Zoque -zom I L Zou -zoo I L Asunción Mixtepec Zapotec -zoq I L Tabasco Zoque -zor I L Rayón Zoque -zos I L Francisco León Zoque -zpa I L Lachiguiri Zapotec -zpb I L Yautepec Zapotec -zpc I L Choapan Zapotec -zpd I L Southeastern Ixtlán Zapotec -zpe I L Petapa Zapotec -zpf I L San Pedro Quiatoni Zapotec -zpg I L Guevea De Humboldt Zapotec -zph I L Totomachapan Zapotec -zpi I L Santa María Quiegolani Zapotec -zpj I L Quiavicuzas Zapotec -zpk I L Tlacolulita Zapotec -zpl I L Lachixío Zapotec -zpm I L Mixtepec Zapotec -zpn I L Santa Inés Yatzechi Zapotec -zpo I L Amatlán Zapotec -zpp I L El Alto Zapotec -zpq I L Zoogocho Zapotec -zpr I L Santiago Xanica Zapotec -zps I L Coatlán Zapotec -zpt I L San Vicente Coatlán Zapotec -zpu I L Yalálag Zapotec -zpv I L Chichicapan Zapotec -zpw I L Zaniza Zapotec -zpx I L San Baltazar Loxicha Zapotec -zpy I L Mazaltepec Zapotec -zpz I L Texmelucan Zapotec -zqe I L Qiubei Zhuang -zra I E Kara (Korea) -zrg I L Mirgan -zrn I L Zerenkel -zro I L Záparo -zrp I E Zarphatic -zrs I L Mairasi -zsa I L Sarasira -zsk I A Kaskean -zsl I L Zambian Sign Language -zsm I L Standard Malay -zsr I L Southern Rincon Zapotec -zsu I L Sukurum -zte I L Elotepec Zapotec -ztg I L Xanaguía Zapotec -ztl I L Lapaguía-Guivini Zapotec -ztm I L San Agustín Mixtepec Zapotec -ztn I L Santa Catarina Albarradas Zapotec -ztp I L Loxicha Zapotec -ztq I L Quioquitani-Quierí Zapotec -zts I L Tilquiapan Zapotec -ztt I L Tejalapan Zapotec -ztu I L Güilá Zapotec -ztx I L Zaachila Zapotec -zty I L Yatee Zapotec -zua I L Zeem -zuh I L Tokano -zul zul zul zu I L Zulu -zum I L Kumzari -zun zun zun I L Zuni -zuy I L Zumaya -zwa I L Zay -zxx zxx zxx S S No linguistic content -zyb I L Yongbei Zhuang -zyg I L Yang Zhuang -zyj I L Youjiang Zhuang -zyn I L Yongnan Zhuang -zyp I L Zyphe Chin -zza zza zza M L Zaza +Id Part2B Part2T Part1 Scope Language_Type Ref_Name Comment +aaa I L Ghotuo +aab I L Alumu-Tesu +aac I L Ari +aad I L Amal +aae I L Arbëreshë Albanian +aaf I L Aranadan +aag I L Ambrak +aah I L Abu' Arapesh +aai I L Arifama-Miniafia +aak I L Ankave +aal I L Afade +aam I L Aramanik +aan I L Anambé +aao I L Algerian Saharan Arabic +aap I L Pará Arára +aaq I E Eastern Abnaki +aar aar aar aa I L Afar +aas I L Aasáx +aat I L Arvanitika Albanian +aau I L Abau +aaw I L Solong +aax I L Mandobo Atas +aaz I L Amarasi +aba I L Abé +abb I L Bankon +abc I L Ambala Ayta +abd I L Manide +abe I E Western Abnaki +abf I L Abai Sungai +abg I L Abaga +abh I L Tajiki Arabic +abi I L Abidji +abj I E Aka-Bea +abk abk abk ab I L Abkhazian +abl I L Lampung Nyo +abm I L Abanyom +abn I L Abua +abo I L Abon +abp I L Abellen Ayta +abq I L Abaza +abr I L Abron +abs I L Ambonese Malay +abt I L Ambulas +abu I L Abure +abv I L Baharna Arabic +abw I L Pal +abx I L Inabaknon +aby I L Aneme Wake +abz I L Abui +aca I L Achagua +acb I L Áncá +acd I L Gikyode +ace ace ace I L Achinese +acf I L Saint Lucian Creole French +ach ach ach I L Acoli +aci I E Aka-Cari +ack I E Aka-Kora +acl I E Akar-Bale +acm I L Mesopotamian Arabic +acn I L Achang +acp I L Eastern Acipa +acq I L Ta'izzi-Adeni Arabic +acr I L Achi +acs I E Acroá +act I L Achterhoeks +acu I L Achuar-Shiwiar +acv I L Achumawi +acw I L Hijazi Arabic +acx I L Omani Arabic +acy I L Cypriot Arabic +acz I L Acheron +ada ada ada I L Adangme +adb I L Adabe +add I L Dzodinka +ade I L Adele +adf I L Dhofari Arabic +adg I L Andegerebinha +adh I L Adhola +adi I L Adi +adj I L Adioukrou +adl I L Galo +adn I L Adang +ado I L Abu +adp I L Adap +adq I L Adangbe +adr I L Adonara +ads I L Adamorobe Sign Language +adt I L Adnyamathanha +adu I L Aduge +adw I L Amundava +adx I L Amdo Tibetan +ady ady ady I L Adyghe +adz I L Adzera +aea I E Areba +aeb I L Tunisian Arabic +aec I L Saidi Arabic +aed I L Argentine Sign Language +aee I L Northeast Pashayi +aek I L Haeke +ael I L Ambele +aem I L Arem +aen I L Armenian Sign Language +aeq I L Aer +aer I L Eastern Arrernte +aes I E Alsea +aeu I L Akeu +aew I L Ambakich +aey I L Amele +aez I L Aeka +afb I L Gulf Arabic +afd I L Andai +afe I L Putukwam +afg I L Afghan Sign Language +afh afh afh I C Afrihili +afi I L Akrukay +afk I L Nanubae +afn I L Defaka +afo I L Eloyi +afp I L Tapei +afr afr afr af I L Afrikaans +afs I L Afro-Seminole Creole +aft I L Afitti +afu I L Awutu +afz I L Obokuitai +aga I E Aguano +agb I L Legbo +agc I L Agatu +agd I L Agarabi +age I L Angal +agf I L Arguni +agg I L Angor +agh I L Ngelima +agi I L Agariya +agj I L Argobba +agk I L Isarog Agta +agl I L Fembe +agm I L Angaataha +agn I L Agutaynen +ago I L Tainae +agq I L Aghem +agr I L Aguaruna +ags I L Esimbi +agt I L Central Cagayan Agta +agu I L Aguacateco +agv I L Remontado Dumagat +agw I L Kahua +agx I L Aghul +agy I L Southern Alta +agz I L Mt. Iriga Agta +aha I L Ahanta +ahb I L Axamb +ahg I L Qimant +ahh I L Aghu +ahi I L Tiagbamrin Aizi +ahk I L Akha +ahl I L Igo +ahm I L Mobumrin Aizi +ahn I L Àhàn +aho I E Ahom +ahp I L Aproumu Aizi +ahr I L Ahirani +ahs I L Ashe +aht I L Ahtena +aia I L Arosi +aib I L Ainu (China) +aic I L Ainbai +aid I E Alngith +aie I L Amara +aif I L Agi +aig I L Antigua and Barbuda Creole English +aih I L Ai-Cham +aii I L Assyrian Neo-Aramaic +aij I L Lishanid Noshan +aik I L Ake +ail I L Aimele +aim I L Aimol +ain ain ain I L Ainu (Japan) +aio I L Aiton +aip I L Burumakok +aiq I L Aimaq +air I L Airoran +ais I L Nataoran Amis +ait I E Arikem +aiw I L Aari +aix I L Aighon +aiy I L Ali +aja I L Aja (Sudan) +ajg I L Aja (Benin) +aji I L Ajië +ajn I L Andajin +ajp I L South Levantine Arabic +ajt I L Judeo-Tunisian Arabic +aju I L Judeo-Moroccan Arabic +ajw I E Ajawa +ajz I L Amri Karbi +aka aka aka ak M L Akan +akb I L Batak Angkola +akc I L Mpur +akd I L Ukpet-Ehom +ake I L Akawaio +akf I L Akpa +akg I L Anakalangu +akh I L Angal Heneng +aki I L Aiome +akj I E Aka-Jeru +akk akk akk I A Akkadian +akl I L Aklanon +akm I E Aka-Bo +ako I L Akurio +akp I L Siwu +akq I L Ak +akr I L Araki +aks I L Akaselem +akt I L Akolet +aku I L Akum +akv I L Akhvakh +akw I L Akwa +akx I E Aka-Kede +aky I E Aka-Kol +akz I L Alabama +ala I L Alago +alc I L Qawasqar +ald I L Alladian +ale ale ale I L Aleut +alf I L Alege +alh I L Alawa +ali I L Amaimon +alj I L Alangan +alk I L Alak +all I L Allar +alm I L Amblong +aln I L Gheg Albanian +alo I L Larike-Wakasihu +alp I L Alune +alq I L Algonquin +alr I L Alutor +als I L Tosk Albanian +alt alt alt I L Southern Altai +alu I L 'Are'are +alw I L Alaba-K’abeena +alx I L Amol +aly I L Alyawarr +alz I L Alur +ama I E Amanayé +amb I L Ambo +amc I L Amahuaca +ame I L Yanesha' +amf I L Hamer-Banna +amg I L Amurdak +amh amh amh am I L Amharic +ami I L Amis +amj I L Amdang +amk I L Ambai +aml I L War-Jaintia +amm I L Ama (Papua New Guinea) +amn I L Amanab +amo I L Amo +amp I L Alamblak +amq I L Amahai +amr I L Amarakaeri +ams I L Southern Amami-Oshima +amt I L Amto +amu I L Guerrero Amuzgo +amv I L Ambelau +amw I L Western Neo-Aramaic +amx I L Anmatyerre +amy I L Ami +amz I E Atampaya +ana I E Andaqui +anb I E Andoa +anc I L Ngas +and I L Ansus +ane I L Xârâcùù +anf I L Animere +ang ang ang I H Old English (ca. 450-1100) +anh I L Nend +ani I L Andi +anj I L Anor +ank I L Goemai +anl I L Anu-Hkongso Chin +anm I L Anal +ann I L Obolo +ano I L Andoque +anp anp anp I L Angika +anq I L Jarawa (India) +anr I L Andh +ans I E Anserma +ant I L Antakarinya +anu I L Anuak +anv I L Denya +anw I L Anaang +anx I L Andra-Hus +any I L Anyin +anz I L Anem +aoa I L Angolar +aob I L Abom +aoc I L Pemon +aod I L Andarum +aoe I L Angal Enen +aof I L Bragat +aog I L Angoram +aoh I E Arma +aoi I L Anindilyakwa +aoj I L Mufian +aok I L Arhö +aol I L Alor +aom I L Ömie +aon I L Bumbita Arapesh +aor I E Aore +aos I L Taikat +aot I L A'tong +aou I L A'ou +aox I L Atorada +aoz I L Uab Meto +apb I L Sa'a +apc I L North Levantine Arabic +apd I L Sudanese Arabic +ape I L Bukiyip +apf I L Pahanan Agta +apg I L Ampanang +aph I L Athpariya +api I L Apiaká +apj I L Jicarilla Apache +apk I L Kiowa Apache +apl I L Lipan Apache +apm I L Mescalero-Chiricahua Apache +apn I L Apinayé +apo I L Ambul +app I L Apma +apq I L A-Pucikwar +apr I L Arop-Lokep +aps I L Arop-Sissano +apt I L Apatani +apu I L Apurinã +apv I E Alapmunte +apw I L Western Apache +apx I L Aputai +apy I L Apalaí +apz I L Safeyoka +aqc I L Archi +aqd I L Ampari Dogon +aqg I L Arigidi +aqm I L Atohwaim +aqn I L Northern Alta +aqp I E Atakapa +aqr I L Arhâ +aqz I L Akuntsu +ara ara ara ar M L Arabic +arb I L Standard Arabic +arc arc arc I A Official Aramaic (700-300 BCE) +ard I E Arabana +are I L Western Arrarnta +arg arg arg an I L Aragonese +arh I L Arhuaco +ari I L Arikara +arj I E Arapaso +ark I L Arikapú +arl I L Arabela +arn arn arn I L Mapudungun +aro I L Araona +arp arp arp I L Arapaho +arq I L Algerian Arabic +arr I L Karo (Brazil) +ars I L Najdi Arabic +aru I E Aruá (Amazonas State) +arv I L Arbore +arw arw arw I L Arawak +arx I L Aruá (Rodonia State) +ary I L Moroccan Arabic +arz I L Egyptian Arabic +asa I L Asu (Tanzania) +asb I L Assiniboine +asc I L Casuarina Coast Asmat +asd I L Asas +ase I L American Sign Language +asf I L Australian Sign Language +asg I L Cishingini +ash I E Abishira +asi I L Buruwai +asj I L Sari +ask I L Ashkun +asl I L Asilulu +asm asm asm as I L Assamese +asn I L Xingú Asuriní +aso I L Dano +asp I L Algerian Sign Language +asq I L Austrian Sign Language +asr I L Asuri +ass I L Ipulo +ast ast ast I L Asturian +asu I L Tocantins Asurini +asv I L Asoa +asw I L Australian Aborigines Sign Language +asx I L Muratayak +asy I L Yaosakor Asmat +asz I L As +ata I L Pele-Ata +atb I L Zaiwa +atc I E Atsahuaca +atd I L Ata Manobo +ate I L Atemble +atg I L Ivbie North-Okpela-Arhe +ati I L Attié +atj I L Atikamekw +atk I L Ati +atl I L Mt. Iraya Agta +atm I L Ata +atn I L Ashtiani +ato I L Atong +atp I L Pudtol Atta +atq I L Aralle-Tabulahan +atr I L Waimiri-Atroari +ats I L Gros Ventre +att I L Pamplona Atta +atu I L Reel +atv I L Northern Altai +atw I L Atsugewi +atx I L Arutani +aty I L Aneityum +atz I L Arta +aua I L Asumboa +aub I L Alugu +auc I L Waorani +aud I L Anuta +aue I L =/Kx'au//'ein +aug I L Aguna +auh I L Aushi +aui I L Anuki +auj I L Awjilah +auk I L Heyo +aul I L Aulua +aum I L Asu (Nigeria) +aun I L Molmo One +auo I E Auyokawa +aup I L Makayam +auq I L Anus +aur I L Aruek +aut I L Austral +auu I L Auye +auw I L Awyi +aux I E Aurá +auy I L Awiyaana +auz I L Uzbeki Arabic +ava ava ava av I L Avaric +avb I L Avau +avd I L Alviri-Vidari +ave ave ave ae I A Avestan +avi I L Avikam +avk I C Kotava +avl I L Eastern Egyptian Bedawi Arabic +avm I E Angkamuthi +avn I L Avatime +avo I E Agavotaguerra +avs I E Aushiri +avt I L Au +avu I L Avokaya +avv I L Avá-Canoeiro +awa awa awa I L Awadhi +awb I L Awa (Papua New Guinea) +awc I L Cicipu +awe I L Awetí +awg I E Anguthimri +awh I L Awbono +awi I L Aekyom +awk I E Awabakal +awm I L Arawum +awn I L Awngi +awo I L Awak +awr I L Awera +aws I L South Awyu +awt I L Araweté +awu I L Central Awyu +awv I L Jair Awyu +aww I L Awun +awx I L Awara +awy I L Edera Awyu +axb I E Abipon +axe I E Ayerrerenge +axg I E Mato Grosso Arára +axk I L Yaka (Central African Republic) +axl I E Lower Southern Aranda +axm I H Middle Armenian +axx I L Xârâgurè +aya I L Awar +ayb I L Ayizo Gbe +ayc I L Southern Aymara +ayd I E Ayabadhu +aye I L Ayere +ayg I L Ginyanga +ayh I L Hadrami Arabic +ayi I L Leyigha +ayk I L Akuku +ayl I L Libyan Arabic +aym aym aym ay M L Aymara +ayn I L Sanaani Arabic +ayo I L Ayoreo +ayp I L North Mesopotamian Arabic +ayq I L Ayi (Papua New Guinea) +ayr I L Central Aymara +ays I L Sorsogon Ayta +ayt I L Magbukun Ayta +ayu I L Ayu +ayy I E Tayabas Ayta +ayz I L Mai Brat +aza I L Azha +azb I L South Azerbaijani +azd I L Eastern Durango Nahuatl +aze aze aze az M L Azerbaijani +azg I L San Pedro Amuzgos Amuzgo +azj I L North Azerbaijani +azm I L Ipalapa Amuzgo +azn I L Western Durango Nahuatl +azo I L Awing +azt I L Faire Atta +azz I L Highland Puebla Nahuatl +baa I L Babatana +bab I L Bainouk-Gunyuño +bac I L Badui +bae I E Baré +baf I L Nubaca +bag I L Tuki +bah I L Bahamas Creole English +baj I L Barakai +bak bak bak ba I L Bashkir +bal bal bal M L Baluchi +bam bam bam bm I L Bambara +ban ban ban I L Balinese +bao I L Waimaha +bap I L Bantawa +bar I L Bavarian +bas bas bas I L Basa (Cameroon) +bau I L Bada (Nigeria) +bav I L Vengo +baw I L Bambili-Bambui +bax I L Bamun +bay I L Batuley +bba I L Baatonum +bbb I L Barai +bbc I L Batak Toba +bbd I L Bau +bbe I L Bangba +bbf I L Baibai +bbg I L Barama +bbh I L Bugan +bbi I L Barombi +bbj I L Ghomálá' +bbk I L Babanki +bbl I L Bats +bbm I L Babango +bbn I L Uneapa +bbo I L Northern Bobo Madaré +bbp I L West Central Banda +bbq I L Bamali +bbr I L Girawa +bbs I L Bakpinka +bbt I L Mburku +bbu I L Kulung (Nigeria) +bbv I L Karnai +bbw I L Baba +bbx I L Bubia +bby I L Befang +bbz I L Babalia Creole Arabic +bca I L Central Bai +bcb I L Bainouk-Samik +bcc I L Southern Balochi +bcd I L North Babar +bce I L Bamenyam +bcf I L Bamu +bcg I L Baga Binari +bch I L Bariai +bci I L Baoulé +bcj I L Bardi +bck I L Bunaba +bcl I L Central Bikol +bcm I L Bannoni +bcn I L Bali (Nigeria) +bco I L Kaluli +bcp I L Bali (Democratic Republic of Congo) +bcq I L Bench +bcr I L Babine +bcs I L Kohumono +bct I L Bendi +bcu I L Awad Bing +bcv I L Shoo-Minda-Nye +bcw I L Bana +bcy I L Bacama +bcz I L Bainouk-Gunyaamolo +bda I L Bayot +bdb I L Basap +bdc I L Emberá-Baudó +bdd I L Bunama +bde I L Bade +bdf I L Biage +bdg I L Bonggi +bdh I L Baka (Sudan) +bdi I L Burun +bdj I L Bai +bdk I L Budukh +bdl I L Indonesian Bajau +bdm I L Buduma +bdn I L Baldemu +bdo I L Morom +bdp I L Bende +bdq I L Bahnar +bdr I L West Coast Bajau +bds I L Burunge +bdt I L Bokoto +bdu I L Oroko +bdv I L Bodo Parja +bdw I L Baham +bdx I L Budong-Budong +bdy I L Bandjalang +bdz I L Badeshi +bea I L Beaver +beb I L Bebele +bec I L Iceve-Maci +bed I L Bedoanas +bee I L Byangsi +bef I L Benabena +beg I L Belait +beh I L Biali +bei I L Bekati' +bej bej bej I L Beja +bek I L Bebeli +bel bel bel be I L Belarusian +bem bem bem I L Bemba (Zambia) +ben ben ben bn I L Bengali +beo I L Beami +bep I L Besoa +beq I L Beembe +bes I L Besme +bet I L Guiberoua Béte +beu I L Blagar +bev I L Daloa Bété +bew I L Betawi +bex I L Jur Modo +bey I L Beli (Papua New Guinea) +bez I L Bena (Tanzania) +bfa I L Bari +bfb I L Pauri Bareli +bfc I L Northern Bai +bfd I L Bafut +bfe I L Betaf +bff I L Bofi +bfg I L Busang Kayan +bfh I L Blafe +bfi I L British Sign Language +bfj I L Bafanji +bfk I L Ban Khor Sign Language +bfl I L Banda-Ndélé +bfm I L Mmen +bfn I L Bunak +bfo I L Malba Birifor +bfp I L Beba +bfq I L Badaga +bfr I L Bazigar +bfs I L Southern Bai +bft I L Balti +bfu I L Gahri +bfw I L Bondo +bfx I L Bantayanon +bfy I L Bagheli +bfz I L Mahasu Pahari +bga I L Gwamhi-Wuri +bgb I L Bobongko +bgc I L Haryanvi +bgd I L Rathwi Bareli +bge I L Bauria +bgf I L Bangandu +bgg I L Bugun +bgi I L Giangan +bgj I L Bangolan +bgk I L Bit +bgl I L Bo (Laos) +bgm I L Baga Mboteni +bgn I L Western Balochi +bgo I L Baga Koga +bgp I L Eastern Balochi +bgq I L Bagri +bgr I L Bawm Chin +bgs I L Tagabawa +bgt I L Bughotu +bgu I L Mbongno +bgv I L Warkay-Bipim +bgw I L Bhatri +bgx I L Balkan Gagauz Turkish +bgy I L Benggoi +bgz I L Banggai +bha I L Bharia +bhb I L Bhili +bhc I L Biga +bhd I L Bhadrawahi +bhe I L Bhaya +bhf I L Odiai +bhg I L Binandere +bhh I L Bukharic +bhi I L Bhilali +bhj I L Bahing +bhl I L Bimin +bhm I L Bathari +bhn I L Bohtan Neo-Aramaic +bho bho bho I L Bhojpuri +bhp I L Bima +bhq I L Tukang Besi South +bhr I L Bara Malagasy +bhs I L Buwal +bht I L Bhattiyali +bhu I L Bhunjia +bhv I L Bahau +bhw I L Biak +bhx I L Bhalay +bhy I L Bhele +bhz I L Bada (Indonesia) +bia I L Badimaya +bib I L Bissa +bic I L Bikaru +bid I L Bidiyo +bie I L Bepour +bif I L Biafada +big I L Biangai +bij I L Vaghat-Ya-Bijim-Legeri +bik bik bik M L Bikol +bil I L Bile +bim I L Bimoba +bin bin bin I L Bini +bio I L Nai +bip I L Bila +biq I L Bipi +bir I L Bisorio +bis bis bis bi I L Bislama +bit I L Berinomo +biu I L Biete +biv I L Southern Birifor +biw I L Kol (Cameroon) +bix I L Bijori +biy I L Birhor +biz I L Baloi +bja I L Budza +bjb I E Banggarla +bjc I L Bariji +bje I L Biao-Jiao Mien +bjf I L Barzani Jewish Neo-Aramaic +bjg I L Bidyogo +bjh I L Bahinemo +bji I L Burji +bjj I L Kanauji +bjk I L Barok +bjl I L Bulu (Papua New Guinea) +bjm I L Bajelani +bjn I L Banjar +bjo I L Mid-Southern Banda +bjp I L Fanamaket +bjr I L Binumarien +bjs I L Bajan +bjt I L Balanta-Ganja +bju I L Busuu +bjv I L Bedjond +bjw I L Bakwé +bjx I L Banao Itneg +bjy I E Bayali +bjz I L Baruga +bka I L Kyak +bkc I L Baka (Cameroon) +bkd I L Binukid +bkf I L Beeke +bkg I L Buraka +bkh I L Bakoko +bki I L Baki +bkj I L Pande +bkk I L Brokskat +bkl I L Berik +bkm I L Kom (Cameroon) +bkn I L Bukitan +bko I L Kwa' +bkp I L Boko (Democratic Republic of Congo) +bkq I L Bakairí +bkr I L Bakumpai +bks I L Northern Sorsoganon +bkt I L Boloki +bku I L Buhid +bkv I L Bekwarra +bkw I L Bekwel +bkx I L Baikeno +bky I L Bokyi +bkz I L Bungku +bla bla bla I L Siksika +blb I L Bilua +blc I L Bella Coola +bld I L Bolango +ble I L Balanta-Kentohe +blf I L Buol +blg I L Balau +blh I L Kuwaa +bli I L Bolia +blj I L Bolongan +blk I L Pa'o Karen +bll I E Biloxi +blm I L Beli (Sudan) +bln I L Southern Catanduanes Bikol +blo I L Anii +blp I L Blablanga +blq I L Baluan-Pam +blr I L Blang +bls I L Balaesang +blt I L Tai Dam +blv I L Bolo +blw I L Balangao +blx I L Mag-Indi Ayta +bly I L Notre +blz I L Balantak +bma I L Lame +bmb I L Bembe +bmc I L Biem +bmd I L Baga Manduri +bme I L Limassa +bmf I L Bom +bmg I L Bamwe +bmh I L Kein +bmi I L Bagirmi +bmj I L Bote-Majhi +bmk I L Ghayavi +bml I L Bomboli +bmm I L Northern Betsimisaraka Malagasy +bmn I E Bina (Papua New Guinea) +bmo I L Bambalang +bmp I L Bulgebi +bmq I L Bomu +bmr I L Muinane +bms I L Bilma Kanuri +bmt I L Biao Mon +bmu I L Somba-Siawari +bmv I L Bum +bmw I L Bomwali +bmx I L Baimak +bmy I L Bemba (Democratic Republic of Congo) +bmz I L Baramu +bna I L Bonerate +bnb I L Bookan +bnc M L Bontok +bnd I L Banda (Indonesia) +bne I L Bintauna +bnf I L Masiwang +bng I L Benga +bni I L Bangi +bnj I L Eastern Tawbuid +bnk I L Bierebo +bnl I L Boon +bnm I L Batanga +bnn I L Bunun +bno I L Bantoanon +bnp I L Bola +bnq I L Bantik +bnr I L Butmas-Tur +bns I L Bundeli +bnu I L Bentong +bnv I L Bonerif +bnw I L Bisis +bnx I L Bangubangu +bny I L Bintulu +bnz I L Beezen +boa I L Bora +bob I L Aweer +bod tib bod bo I L Tibetan +boe I L Mundabli +bof I L Bolon +bog I L Bamako Sign Language +boh I L Boma +boi I E Barbareño +boj I L Anjam +bok I L Bonjo +bol I L Bole +bom I L Berom +bon I L Bine +boo I L Tiemacèwè Bozo +bop I L Bonkiman +boq I L Bogaya +bor I L Borôro +bos bos bos bs I L Bosnian +bot I L Bongo +bou I L Bondei +bov I L Tuwuli +bow I E Rema +box I L Buamu +boy I L Bodo (Central African Republic) +boz I L Tiéyaxo Bozo +bpa I L Daakaka +bpb I E Barbacoas +bpd I L Banda-Banda +bpg I L Bonggo +bph I L Botlikh +bpi I L Bagupi +bpj I L Binji +bpk I L Orowe +bpl I L Broome Pearling Lugger Pidgin +bpm I L Biyom +bpn I L Dzao Min +bpo I L Anasi +bpp I L Kaure +bpq I L Banda Malay +bpr I L Koronadal Blaan +bps I L Sarangani Blaan +bpt I E Barrow Point +bpu I L Bongu +bpv I L Bian Marind +bpw I L Bo (Papua New Guinea) +bpx I L Palya Bareli +bpy I L Bishnupriya +bpz I L Bilba +bqa I L Tchumbuli +bqb I L Bagusa +bqc I L Boko (Benin) +bqd I L Bung +bqf I E Baga Kaloum +bqg I L Bago-Kusuntu +bqh I L Baima +bqi I L Bakhtiari +bqj I L Bandial +bqk I L Banda-Mbrès +bql I L Bilakura +bqm I L Wumboko +bqn I L Bulgarian Sign Language +bqo I L Balo +bqp I L Busa +bqq I L Biritai +bqr I L Burusu +bqs I L Bosngun +bqt I L Bamukumbit +bqu I L Boguru +bqv I L Koro Wachi +bqw I L Buru (Nigeria) +bqx I L Baangi +bqy I L Bengkala Sign Language +bqz I L Bakaka +bra bra bra I L Braj +brb I L Lave +brc I E Berbice Creole Dutch +brd I L Baraamu +bre bre bre br I L Breton +brf I L Bera +brg I L Baure +brh I L Brahui +bri I L Mokpwe +brj I L Bieria +brk I E Birked +brl I L Birwa +brm I L Barambu +brn I L Boruca +bro I L Brokkat +brp I L Barapasi +brq I L Breri +brr I L Birao +brs I L Baras +brt I L Bitare +bru I L Eastern Bru +brv I L Western Bru +brw I L Bellari +brx I L Bodo (India) +bry I L Burui +brz I L Bilbil +bsa I L Abinomn +bsb I L Brunei Bisaya +bsc I L Bassari +bse I L Wushi +bsf I L Bauchi +bsg I L Bashkardi +bsh I L Kati +bsi I L Bassossi +bsj I L Bangwinji +bsk I L Burushaski +bsl I E Basa-Gumna +bsm I L Busami +bsn I L Barasana-Eduria +bso I L Buso +bsp I L Baga Sitemu +bsq I L Bassa +bsr I L Bassa-Kontagora +bss I L Akoose +bst I L Basketo +bsu I L Bahonsuai +bsv I E Baga Sobané +bsw I L Baiso +bsx I L Yangkam +bsy I L Sabah Bisaya +bta I L Bata +btc I L Bati (Cameroon) +btd I L Batak Dairi +bte I E Gamo-Ningi +btf I L Birgit +btg I L Gagnoa Bété +bth I L Biatah Bidayuh +bti I L Burate +btj I L Bacanese Malay +btl I L Bhatola +btm I L Batak Mandailing +btn I L Ratagnon +bto I L Rinconada Bikol +btp I L Budibud +btq I L Batek +btr I L Baetora +bts I L Batak Simalungun +btt I L Bete-Bendi +btu I L Batu +btv I L Bateri +btw I L Butuanon +btx I L Batak Karo +bty I L Bobot +btz I L Batak Alas-Kluet +bua bua bua M L Buriat +bub I L Bua +buc I L Bushi +bud I L Ntcham +bue I E Beothuk +buf I L Bushoong +bug bug bug I L Buginese +buh I L Younuo Bunu +bui I L Bongili +buj I L Basa-Gurmana +buk I L Bugawac +bul bul bul bg I L Bulgarian +bum I L Bulu (Cameroon) +bun I L Sherbro +buo I L Terei +bup I L Busoa +buq I L Brem +bus I L Bokobaru +but I L Bungain +buu I L Budu +buv I L Bun +buw I L Bubi +bux I L Boghom +buy I L Bullom So +buz I L Bukwen +bva I L Barein +bvb I L Bube +bvc I L Baelelea +bvd I L Baeggu +bve I L Berau Malay +bvf I L Boor +bvg I L Bonkeng +bvh I L Bure +bvi I L Belanda Viri +bvj I L Baan +bvk I L Bukat +bvl I L Bolivian Sign Language +bvm I L Bamunka +bvn I L Buna +bvo I L Bolgo +bvp I L Bumang +bvq I L Birri +bvr I L Burarra +bvt I L Bati (Indonesia) +bvu I L Bukit Malay +bvv I E Baniva +bvw I L Boga +bvx I L Dibole +bvy I L Baybayanon +bvz I L Bauzi +bwa I L Bwatoo +bwb I L Namosi-Naitasiri-Serua +bwc I L Bwile +bwd I L Bwaidoka +bwe I L Bwe Karen +bwf I L Boselewa +bwg I L Barwe +bwh I L Bishuo +bwi I L Baniwa +bwj I L Láá Láá Bwamu +bwk I L Bauwaki +bwl I L Bwela +bwm I L Biwat +bwn I L Wunai Bunu +bwo I L Boro (Ethiopia) +bwp I L Mandobo Bawah +bwq I L Southern Bobo Madaré +bwr I L Bura-Pabir +bws I L Bomboma +bwt I L Bafaw-Balong +bwu I L Buli (Ghana) +bww I L Bwa +bwx I L Bu-Nao Bunu +bwy I L Cwi Bwamu +bwz I L Bwisi +bxa I L Tairaha +bxb I L Belanda Bor +bxc I L Molengue +bxd I L Pela +bxe I L Birale +bxf I L Bilur +bxg I L Bangala +bxh I L Buhutu +bxi I E Pirlatapa +bxj I L Bayungu +bxk I L Bukusu +bxl I L Jalkunan +bxm I L Mongolia Buriat +bxn I L Burduna +bxo I L Barikanchi +bxp I L Bebil +bxq I L Beele +bxr I L Russia Buriat +bxs I L Busam +bxu I L China Buriat +bxv I L Berakou +bxw I L Bankagooma +bxx I L Borna (Democratic Republic of Congo) +bxz I L Binahari +bya I L Batak +byb I L Bikya +byc I L Ubaghara +byd I L Benyadu' +bye I L Pouye +byf I L Bete +byg I E Baygo +byh I L Bhujel +byi I L Buyu +byj I L Bina (Nigeria) +byk I L Biao +byl I L Bayono +bym I L Bidyara +byn byn byn I L Bilin +byo I L Biyo +byp I L Bumaji +byq I E Basay +byr I L Baruya +bys I L Burak +byt I E Berti +byv I L Medumba +byw I L Belhariya +byx I L Qaqet +byy I L Buya +byz I L Banaro +bza I L Bandi +bzb I L Andio +bzc I L Southern Betsimisaraka Malagasy +bzd I L Bribri +bze I L Jenaama Bozo +bzf I L Boikin +bzg I L Babuza +bzh I L Mapos Buang +bzi I L Bisu +bzj I L Belize Kriol English +bzk I L Nicaragua Creole English +bzl I L Boano (Sulawesi) +bzm I L Bolondo +bzn I L Boano (Maluku) +bzo I L Bozaba +bzp I L Kemberano +bzq I L Buli (Indonesia) +bzr I E Biri +bzs I L Brazilian Sign Language +bzt I C Brithenig +bzu I L Burmeso +bzv I L Naami +bzw I L Basa (Nigeria) +bzx I L Kɛlɛngaxo Bozo +bzy I L Obanliku +bzz I L Evant +caa I L Chortí +cab I L Garifuna +cac I L Chuj +cad cad cad I L Caddo +cae I L Lehar +caf I L Southern Carrier +cag I L Nivaclé +cah I L Cahuarano +caj I E Chané +cak I L Kaqchikel +cal I L Carolinian +cam I L Cemuhî +can I L Chambri +cao I L Chácobo +cap I L Chipaya +caq I L Car Nicobarese +car car car I L Galibi Carib +cas I L Tsimané +cat cat cat ca I L Catalan +cav I L Cavineña +caw I L Callawalla +cax I L Chiquitano +cay I L Cayuga +caz I E Canichana +cbb I L Cabiyarí +cbc I L Carapana +cbd I L Carijona +cbe I E Chipiajes +cbg I L Chimila +cbh I E Cagua +cbi I L Chachi +cbj I L Ede Cabe +cbk I L Chavacano +cbl I L Bualkhaw Chin +cbn I L Nyahkur +cbo I L Izora +cbr I L Cashibo-Cacataibo +cbs I L Cashinahua +cbt I L Chayahuita +cbu I L Candoshi-Shapra +cbv I L Cacua +cbw I L Kinabalian +cby I L Carabayo +cca I E Cauca +ccc I L Chamicuro +ccd I L Cafundo Creole +cce I L Chopi +ccg I L Samba Daka +cch I L Atsam +ccj I L Kasanga +ccl I L Cutchi-Swahili +ccm I L Malaccan Creole Malay +cco I L Comaltepec Chinantec +ccp I L Chakma +ccr I E Cacaopera +cda I L Choni +cde I L Chenchu +cdf I L Chiru +cdg I L Chamari +cdh I L Chambeali +cdi I L Chodri +cdj I L Churahi +cdm I L Chepang +cdn I L Chaudangsi +cdo I L Min Dong Chinese +cdr I L Cinda-Regi-Tiyal +cds I L Chadian Sign Language +cdy I L Chadong +cdz I L Koda +cea I E Lower Chehalis +ceb ceb ceb I L Cebuano +ceg I L Chamacoco +cek I L Eastern Khumi Chin +cen I L Cen +ces cze ces cs I L Czech +cet I L Centúúm +cfa I L Dijim-Bwilim +cfd I L Cara +cfg I L Como Karim +cfm I L Falam Chin +cga I L Changriwa +cgc I L Kagayanen +cgg I L Chiga +cgk I L Chocangacakha +cha cha cha ch I L Chamorro +chb chb chb I E Chibcha +chc I E Catawba +chd I L Highland Oaxaca Chontal +che che che ce I L Chechen +chf I L Tabasco Chontal +chg chg chg I E Chagatai +chh I L Chinook +chj I L Ojitlán Chinantec +chk chk chk I L Chuukese +chl I L Cahuilla +chm chm chm M L Mari (Russia) +chn chn chn I L Chinook jargon +cho cho cho I L Choctaw +chp chp chp I L Chipewyan +chq I L Quiotepec Chinantec +chr chr chr I L Cherokee +cht I E Cholón +chu chu chu cu I A Church Slavic +chv chv chv cv I L Chuvash +chw I L Chuwabu +chx I L Chantyal +chy chy chy I L Cheyenne +chz I L Ozumacín Chinantec +cia I L Cia-Cia +cib I L Ci Gbe +cic I L Chickasaw +cid I E Chimariko +cie I L Cineni +cih I L Chinali +cik I L Chitkuli Kinnauri +cim I L Cimbrian +cin I L Cinta Larga +cip I L Chiapanec +cir I L Tiri +ciw I L Chippewa +ciy I L Chaima +cja I L Western Cham +cje I L Chru +cjh I E Upper Chehalis +cji I L Chamalal +cjk I L Chokwe +cjm I L Eastern Cham +cjn I L Chenapian +cjo I L Ashéninka Pajonal +cjp I L Cabécar +cjs I L Shor +cjv I L Chuave +cjy I L Jinyu Chinese +ckb I L Central Kurdish +ckh I L Chak +ckl I L Cibak +ckn I L Kaang Chin +cko I L Anufo +ckq I L Kajakse +ckr I L Kairak +cks I L Tayo +ckt I L Chukot +cku I L Koasati +ckv I L Kavalan +ckx I L Caka +cky I L Cakfem-Mushere +ckz I L Cakchiquel-Quiché Mixed Language +cla I L Ron +clc I L Chilcotin +cld I L Chaldean Neo-Aramaic +cle I L Lealao Chinantec +clh I L Chilisso +cli I L Chakali +clj I L Laitu Chin +clk I L Idu-Mishmi +cll I L Chala +clm I L Clallam +clo I L Lowland Oaxaca Chontal +clt I L Lautu Chin +clu I L Caluyanun +clw I L Chulym +cly I L Eastern Highland Chatino +cma I L Maa +cme I L Cerma +cmg I H Classical Mongolian +cmi I L Emberá-Chamí +cml I L Campalagian +cmm I E Michigamea +cmn I L Mandarin Chinese +cmo I L Central Mnong +cmr I L Mro-Khimi Chin +cms I A Messapic +cmt I L Camtho +cna I L Changthang +cnb I L Chinbon Chin +cnc I L Côông +cng I L Northern Qiang +cnh I L Haka Chin +cni I L Asháninka +cnk I L Khumi Chin +cnl I L Lalana Chinantec +cno I L Con +cns I L Central Asmat +cnt I L Tepetotutla Chinantec +cnu I L Chenoua +cnw I L Ngawn Chin +cnx I H Middle Cornish +coa I L Cocos Islands Malay +cob I E Chicomuceltec +coc I L Cocopa +cod I L Cocama-Cocamilla +coe I L Koreguaje +cof I L Colorado +cog I L Chong +coh I L Chonyi-Dzihana-Kauma +coj I E Cochimi +cok I L Santa Teresa Cora +col I L Columbia-Wenatchi +com I L Comanche +con I L Cofán +coo I L Comox +cop cop cop I E Coptic +coq I E Coquille +cor cor cor kw I L Cornish +cos cos cos co I L Corsican +cot I L Caquinte +cou I L Wamey +cov I L Cao Miao +cow I E Cowlitz +cox I L Nanti +coy I E Coyaima +coz I L Chochotec +cpa I L Palantla Chinantec +cpb I L Ucayali-Yurúa Ashéninka +cpc I L Ajyíninka Apurucayali +cpg I E Cappadocian Greek +cpi I L Chinese Pidgin English +cpn I L Cherepon +cpo I L Kpeego +cps I L Capiznon +cpu I L Pichis Ashéninka +cpx I L Pu-Xian Chinese +cpy I L South Ucayali Ashéninka +cqd I L Chuanqiandian Cluster Miao +cqu I L Chilean Quechua +cra I L Chara +crb I E Island Carib +crc I L Lonwolwol +crd I L Coeur d'Alene +cre cre cre cr M L Cree +crf I E Caramanta +crg I L Michif +crh crh crh I L Crimean Tatar +cri I L Sãotomense +crj I L Southern East Cree +crk I L Plains Cree +crl I L Northern East Cree +crm I L Moose Cree +crn I L El Nayar Cora +cro I L Crow +crq I L Iyo'wujwa Chorote +crr I E Carolina Algonquian +crs I L Seselwa Creole French +crt I L Iyojwa'ja Chorote +crv I L Chaura +crw I L Chrau +crx I L Carrier +cry I L Cori +crz I E Cruzeño +csa I L Chiltepec Chinantec +csb csb csb I L Kashubian +csc I L Catalan Sign Language +csd I L Chiangmai Sign Language +cse I L Czech Sign Language +csf I L Cuba Sign Language +csg I L Chilean Sign Language +csh I L Asho Chin +csi I E Coast Miwok +csj I L Songlai Chin +csk I L Jola-Kasa +csl I L Chinese Sign Language +csm I L Central Sierra Miwok +csn I L Colombian Sign Language +cso I L Sochiapam Chinantec +csq I L Croatia Sign Language +csr I L Costa Rican Sign Language +css I E Southern Ohlone +cst I L Northern Ohlone +csv I L Sumtu Chin +csw I L Swampy Cree +csy I L Siyin Chin +csz I L Coos +cta I L Tataltepec Chatino +ctc I L Chetco +ctd I L Tedim Chin +cte I L Tepinapa Chinantec +ctg I L Chittagonian +cth I L Thaiphum Chin +ctl I L Tlacoatzintepec Chinantec +ctm I E Chitimacha +ctn I L Chhintange +cto I L Emberá-Catío +ctp I L Western Highland Chatino +cts I L Northern Catanduanes Bikol +ctt I L Wayanad Chetti +ctu I L Chol +ctz I L Zacatepec Chatino +cua I L Cua +cub I L Cubeo +cuc I L Usila Chinantec +cug I L Cung +cuh I L Chuka +cui I L Cuiba +cuj I L Mashco Piro +cuk I L San Blas Kuna +cul I L Culina +cum I E Cumeral +cuo I E Cumanagoto +cup I E Cupeño +cuq I L Cun +cur I L Chhulung +cut I L Teutila Cuicatec +cuu I L Tai Ya +cuv I L Cuvok +cuw I L Chukwa +cux I L Tepeuxila Cuicatec +cvg I L Chug +cvn I L Valle Nacional Chinantec +cwa I L Kabwa +cwb I L Maindo +cwd I L Woods Cree +cwe I L Kwere +cwg I L Chewong +cwt I L Kuwaataay +cya I L Nopala Chatino +cyb I E Cayubaba +cym wel cym cy I L Welsh +cyo I L Cuyonon +czh I L Huizhou Chinese +czk I E Knaanic +czn I L Zenzontepec Chatino +czo I L Min Zhong Chinese +czt I L Zotung Chin +daa I L Dangaléat +dac I L Dambi +dad I L Marik +dae I L Duupa +dag I L Dagbani +dah I L Gwahatike +dai I L Day +daj I L Dar Fur Daju +dak dak dak I L Dakota +dal I L Dahalo +dam I L Damakawa +dan dan dan da I L Danish +dao I L Daai Chin +daq I L Dandami Maria +dar dar dar I L Dargwa +das I L Daho-Doo +dau I L Dar Sila Daju +dav I L Taita +daw I L Davawenyo +dax I L Dayi +daz I L Dao +dba I L Bangime +dbb I L Deno +dbd I L Dadiya +dbe I L Dabe +dbf I L Edopi +dbg I L Dogul Dom Dogon +dbi I L Doka +dbj I L Ida'an +dbl I L Dyirbal +dbm I L Duguri +dbn I L Duriankere +dbo I L Dulbu +dbp I L Duwai +dbq I L Daba +dbr I L Dabarre +dbt I L Ben Tey Dogon +dbu I L Bondum Dom Dogon +dbv I L Dungu +dbw I L Bankan Tey Dogon +dby I L Dibiyaso +dcc I L Deccan +dcr I E Negerhollands +dda I E Dadi Dadi +ddd I L Dongotono +dde I L Doondo +ddg I L Fataluku +ddi I L West Goodenough +ddj I L Jaru +ddn I L Dendi (Benin) +ddo I L Dido +ddr I E Dhudhuroa +dds I L Donno So Dogon +ddw I L Dawera-Daweloor +dec I L Dagik +ded I L Dedua +dee I L Dewoin +def I L Dezfuli +deg I L Degema +deh I L Dehwari +dei I L Demisa +dek I L Dek +del del del M L Delaware +dem I L Dem +den den den M L Slave (Athapascan) +dep I E Pidgin Delaware +deq I L Dendi (Central African Republic) +der I L Deori +des I L Desano +deu ger deu de I L German +dev I L Domung +dez I L Dengese +dga I L Southern Dagaare +dgb I L Bunoge Dogon +dgc I L Casiguran Dumagat Agta +dgd I L Dagaari Dioula +dge I L Degenan +dgg I L Doga +dgh I L Dghwede +dgi I L Northern Dagara +dgk I L Dagba +dgl I L Andaandi +dgn I E Dagoman +dgo I L Dogri (individual language) +dgr dgr dgr I L Dogrib +dgs I L Dogoso +dgt I E Ndra'ngith +dgu I L Degaru +dgw I E Daungwurrung +dgx I L Doghoro +dgz I L Daga +dhd I L Dhundari +dhg I L Djangu +dhi I L Dhimal +dhl I L Dhalandji +dhm I L Zemba +dhn I L Dhanki +dho I L Dhodia +dhr I L Dhargari +dhs I L Dhaiso +dhu I E Dhurga +dhv I L Dehu +dhw I L Dhanwar (Nepal) +dhx I L Dhungaloo +dia I L Dia +dib I L South Central Dinka +dic I L Lakota Dida +did I L Didinga +dif I E Dieri +dig I L Digo +dih I L Kumiai +dii I L Dimbong +dij I L Dai +dik I L Southwestern Dinka +dil I L Dilling +dim I L Dime +din din din M L Dinka +dio I L Dibo +dip I L Northeastern Dinka +diq I L Dimli (individual language) +dir I L Dirim +dis I L Dimasa +dit I E Dirari +diu I L Diriku +div div div dv I L Dhivehi +diw I L Northwestern Dinka +dix I L Dixon Reef +diy I L Diuwe +diz I L Ding +dja I E Djadjawurrung +djb I L Djinba +djc I L Dar Daju Daju +djd I L Djamindjung +dje I L Zarma +djf I E Djangun +dji I L Djinang +djj I L Djeebbana +djk I L Eastern Maroon Creole +djm I L Jamsay Dogon +djn I L Djauan +djo I L Jangkang +djr I L Djambarrpuyngu +dju I L Kapriman +djw I E Djawi +dka I L Dakpakha +dkk I L Dakka +dkr I L Kuijau +dks I L Southeastern Dinka +dkx I L Mazagway +dlg I L Dolgan +dlk I L Dahalik +dlm I E Dalmatian +dln I L Darlong +dma I L Duma +dmb I L Mombo Dogon +dmc I L Gavak +dmd I E Madhi Madhi +dme I L Dugwor +dmg I L Upper Kinabatangan +dmk I L Domaaki +dml I L Dameli +dmm I L Dama +dmo I L Kemedzung +dmr I L East Damar +dms I L Dampelas +dmu I L Dubu +dmv I L Dumpas +dmw I L Mudburra +dmx I L Dema +dmy I L Demta +dna I L Upper Grand Valley Dani +dnd I L Daonda +dne I L Ndendeule +dng I L Dungan +dni I L Lower Grand Valley Dani +dnj I L Dan +dnk I L Dengka +dnn I L Dzùùngoo +dnr I L Danaru +dnt I L Mid Grand Valley Dani +dnu I L Danau +dnv I L Danu +dnw I L Western Dani +dny I L Dení +doa I L Dom +dob I L Dobu +doc I L Northern Dong +doe I L Doe +dof I L Domu +doh I L Dong +doi doi doi M L Dogri (macrolanguage) +dok I L Dondo +dol I L Doso +don I L Toura (Papua New Guinea) +doo I L Dongo +dop I L Lukpa +doq I L Dominican Sign Language +dor I L Dori'o +dos I L Dogosé +dot I L Dass +dov I L Dombe +dow I L Doyayo +dox I L Bussa +doy I L Dompo +doz I L Dorze +dpp I L Papar +drb I L Dair +drc I L Minderico +drd I L Darmiya +dre I L Dolpo +drg I L Rungus +dri I L C'lela +drl I L Paakantyi +drn I L West Damar +dro I L Daro-Matu Melanau +drq I E Dura +drr I E Dororo +drs I L Gedeo +drt I L Drents +dru I L Rukai +dry I L Darai +dsb dsb dsb I L Lower Sorbian +dse I L Dutch Sign Language +dsh I L Daasanach +dsi I L Disa +dsl I L Danish Sign Language +dsn I L Dusner +dso I L Desiya +dsq I L Tadaksahak +dta I L Daur +dtb I L Labuk-Kinabatangan Kadazan +dtd I L Ditidaht +dth I E Adithinngithigh +dti I L Ana Tinga Dogon +dtk I L Tene Kan Dogon +dtm I L Tomo Kan Dogon +dto I L Tommo So Dogon +dtp I L Central Dusun +dtr I L Lotud +dts I L Toro So Dogon +dtt I L Toro Tegu Dogon +dtu I L Tebul Ure Dogon +dty I L Dotyali +dua dua dua I L Duala +dub I L Dubli +duc I L Duna +dud I L Hun-Saare +due I L Umiray Dumaget Agta +duf I L Dumbea +dug I L Duruma +duh I L Dungra Bhil +dui I L Dumun +duj I L Dhuwal +duk I L Uyajitaya +dul I L Alabat Island Agta +dum dum dum I H Middle Dutch (ca. 1050-1350) +dun I L Dusun Deyah +duo I L Dupaninan Agta +dup I L Duano +duq I L Dusun Malang +dur I L Dii +dus I L Dumi +duu I L Drung +duv I L Duvle +duw I L Dusun Witu +dux I L Duungooma +duy I E Dicamay Agta +duz I E Duli +dva I L Duau +dwa I L Diri +dwr I L Dawro +dws I C Dutton World Speedwords +dww I L Dawawa +dya I L Dyan +dyb I E Dyaberdyaber +dyd I E Dyugun +dyg I E Villa Viciosa Agta +dyi I L Djimini Senoufo +dym I L Yanda Dom Dogon +dyn I L Dyangadi +dyo I L Jola-Fonyi +dyu dyu dyu I L Dyula +dyy I L Dyaabugay +dza I L Tunzu +dzd I L Daza +dze I E Djiwarli +dzg I L Dazaga +dzl I L Dzalakha +dzn I L Dzando +dzo dzo dzo dz I L Dzongkha +eaa I E Karenggapa +ebg I L Ebughu +ebk I L Eastern Bontok +ebo I L Teke-Ebo +ebr I L Ebrié +ebu I L Embu +ecr I A Eteocretan +ecs I L Ecuadorian Sign Language +ecy I A Eteocypriot +eee I L E +efa I L Efai +efe I L Efe +efi efi efi I L Efik +ega I L Ega +egl I L Emilian +ego I L Eggon +egy egy egy I A Egyptian (Ancient) +ehu I L Ehueun +eip I L Eipomek +eit I L Eitiep +eiv I L Askopan +eja I L Ejamat +eka eka eka I L Ekajuk +ekc I E Eastern Karnic +eke I L Ekit +ekg I L Ekari +eki I L Eki +ekk I L Standard Estonian +ekl I L Kol (Bangladesh) +ekm I L Elip +eko I L Koti +ekp I L Ekpeye +ekr I L Yace +eky I L Eastern Kayah +ele I L Elepi +elh I L El Hugeirat +eli I E Nding +elk I L Elkei +ell gre ell el I L Modern Greek (1453-) +elm I L Eleme +elo I L El Molo +elu I L Elu +elx elx elx I A Elamite +ema I L Emai-Iuleha-Ora +emb I L Embaloh +eme I L Emerillon +emg I L Eastern Meohang +emi I L Mussau-Emira +emk I L Eastern Maninkakan +emm I E Mamulique +emn I L Eman +emo I E Emok +emp I L Northern Emberá +ems I L Pacific Gulf Yupik +emu I L Eastern Muria +emw I L Emplawas +emx I L Erromintxela +emy I E Epigraphic Mayan +ena I L Apali +enb I L Markweeta +enc I L En +end I L Ende +enf I L Forest Enets +eng eng eng en I L English +enh I L Tundra Enets +enm enm enm I H Middle English (1100-1500) +enn I L Engenni +eno I L Enggano +enq I L Enga +enr I L Emumu +enu I L Enu +env I L Enwan (Edu State) +enw I L Enwan (Akwa Ibom State) +eot I L Beti (Côte d'Ivoire) +epi I L Epie +epo epo epo eo I C Esperanto +era I L Eravallan +erg I L Sie +erh I L Eruwa +eri I L Ogea +erk I L South Efate +ero I L Horpa +err I E Erre +ers I L Ersu +ert I L Eritai +erw I L Erokwanas +ese I L Ese Ejja +esh I L Eshtehardi +esi I L North Alaskan Inupiatun +esk I L Northwest Alaska Inupiatun +esl I L Egypt Sign Language +esm I E Esuma +esn I L Salvadoran Sign Language +eso I L Estonian Sign Language +esq I E Esselen +ess I L Central Siberian Yupik +est est est et M L Estonian +esu I L Central Yupik +etb I L Etebi +etc I E Etchemin +eth I L Ethiopian Sign Language +etn I L Eton (Vanuatu) +eto I L Eton (Cameroon) +etr I L Edolo +ets I L Yekhee +ett I A Etruscan +etu I L Ejagham +etx I L Eten +etz I L Semimi +eus baq eus eu I L Basque +eve I L Even +evh I L Uvbie +evn I L Evenki +ewe ewe ewe ee I L Ewe +ewo ewo ewo I L Ewondo +ext I L Extremaduran +eya I E Eyak +eyo I L Keiyo +eza I L Ezaa +eze I L Uzekwe +faa I L Fasu +fab I L Fa d'Ambu +fad I L Wagi +faf I L Fagani +fag I L Finongan +fah I L Baissa Fali +fai I L Faiwol +faj I L Faita +fak I L Fang (Cameroon) +fal I L South Fali +fam I L Fam +fan fan fan I L Fang (Equatorial Guinea) +fao fao fao fo I L Faroese +fap I L Palor +far I L Fataleka +fas per fas fa M L Persian +fat fat fat I L Fanti +fau I L Fayu +fax I L Fala +fay I L Southwestern Fars +faz I L Northwestern Fars +fbl I L West Albay Bikol +fcs I L Quebec Sign Language +fer I L Feroge +ffi I L Foia Foia +ffm I L Maasina Fulfulde +fgr I L Fongoro +fia I L Nobiin +fie I L Fyer +fij fij fij fj I L Fijian +fil fil fil I L Filipino +fin fin fin fi I L Finnish +fip I L Fipa +fir I L Firan +fit I L Tornedalen Finnish +fiw I L Fiwaga +fkk I L Kirya-Konzəl +fkv I L Kven Finnish +fla I L Kalispel-Pend d'Oreille +flh I L Foau +fli I L Fali +fll I L North Fali +fln I E Flinders Island +flr I L Fuliiru +fly I L Tsotsitaal +fmp I L Fe'fe' +fmu I L Far Western Muria +fng I L Fanagalo +fni I L Fania +fod I L Foodo +foi I L Foi +fom I L Foma +fon fon fon I L Fon +for I L Fore +fos I E Siraya +fpe I L Fernando Po Creole English +fqs I L Fas +fra fre fra fr I L French +frc I L Cajun French +frd I L Fordata +frk I E Frankish +frm frm frm I H Middle French (ca. 1400-1600) +fro fro fro I H Old French (842-ca. 1400) +frp I L Arpitan +frq I L Forak +frr frr frr I L Northern Frisian +frs frs frs I L Eastern Frisian +frt I L Fortsenal +fry fry fry fy I L Western Frisian +fse I L Finnish Sign Language +fsl I L French Sign Language +fss I L Finland-Swedish Sign Language +fub I L Adamawa Fulfulde +fuc I L Pulaar +fud I L East Futuna +fue I L Borgu Fulfulde +fuf I L Pular +fuh I L Western Niger Fulfulde +fui I L Bagirmi Fulfulde +fuj I L Ko +ful ful ful ff M L Fulah +fum I L Fum +fun I L Fulniô +fuq I L Central-Eastern Niger Fulfulde +fur fur fur I L Friulian +fut I L Futuna-Aniwa +fuu I L Furu +fuv I L Nigerian Fulfulde +fuy I L Fuyug +fvr I L Fur +fwa I L Fwâi +fwe I L Fwe +gaa gaa gaa I L Ga +gab I L Gabri +gac I L Mixed Great Andamanese +gad I L Gaddang +gae I L Guarequena +gaf I L Gende +gag I L Gagauz +gah I L Alekano +gai I L Borei +gaj I L Gadsup +gak I L Gamkonora +gal I L Galolen +gam I L Kandawo +gan I L Gan Chinese +gao I L Gants +gap I L Gal +gaq I L Gata' +gar I L Galeya +gas I L Adiwasi Garasia +gat I L Kenati +gau I L Mudhili Gadaba +gaw I L Nobonob +gax I L Borana-Arsi-Guji Oromo +gay gay gay I L Gayo +gaz I L West Central Oromo +gba gba gba M L Gbaya (Central African Republic) +gbb I L Kaytetye +gbd I L Karadjeri +gbe I L Niksek +gbf I L Gaikundi +gbg I L Gbanziri +gbh I L Defi Gbe +gbi I L Galela +gbj I L Bodo Gadaba +gbk I L Gaddi +gbl I L Gamit +gbm I L Garhwali +gbn I L Mo'da +gbo I L Northern Grebo +gbp I L Gbaya-Bossangoa +gbq I L Gbaya-Bozoum +gbr I L Gbagyi +gbs I L Gbesi Gbe +gbu I L Gagadu +gbv I L Gbanu +gbw I L Gabi-Gabi +gbx I L Eastern Xwla Gbe +gby I L Gbari +gbz I L Zoroastrian Dari +gcc I L Mali +gcd I E Ganggalida +gce I E Galice +gcf I L Guadeloupean Creole French +gcl I L Grenadian Creole English +gcn I L Gaina +gcr I L Guianese Creole French +gct I L Colonia Tovar German +gda I L Gade Lohar +gdb I L Pottangi Ollar Gadaba +gdc I E Gugu Badhun +gdd I L Gedaged +gde I L Gude +gdf I L Guduf-Gava +gdg I L Ga'dang +gdh I L Gadjerawang +gdi I L Gundi +gdj I L Gurdjar +gdk I L Gadang +gdl I L Dirasha +gdm I L Laal +gdn I L Umanakaina +gdo I L Ghodoberi +gdq I L Mehri +gdr I L Wipi +gds I L Ghandruk Sign Language +gdt I E Kungardutyi +gdu I L Gudu +gdx I L Godwari +gea I L Geruma +geb I L Kire +gec I L Gboloo Grebo +ged I L Gade +geg I L Gengle +geh I L Hutterite German +gei I L Gebe +gej I L Gen +gek I L Yiwom +gel I L ut-Ma'in +geq I L Geme +ges I L Geser-Gorom +gew I L Gera +gex I L Garre +gey I L Enya +gez gez gez I A Geez +gfk I L Patpatar +gft I E Gafat +gfx I L Mangetti Dune !Xung +gga I L Gao +ggb I L Gbii +ggd I E Gugadj +gge I L Guragone +ggg I L Gurgula +ggk I E Kungarakany +ggl I L Ganglau +ggm I E Gugu Mini +ggn I L Eastern Gurung +ggo I L Southern Gondi +ggt I L Gitua +ggu I L Gagu +ggw I L Gogodala +gha I L Ghadamès +ghc I E Hiberno-Scottish Gaelic +ghe I L Southern Ghale +ghh I L Northern Ghale +ghk I L Geko Karen +ghl I L Ghulfan +ghn I L Ghanongga +gho I E Ghomara +ghr I L Ghera +ghs I L Guhu-Samane +ght I L Kuke +gia I L Kitja +gib I L Gibanawa +gic I L Gail +gid I L Gidar +gig I L Goaria +gih I L Githabul +gil gil gil I L Gilbertese +gim I L Gimi (Eastern Highlands) +gin I L Hinukh +gip I L Gimi (West New Britain) +giq I L Green Gelao +gir I L Red Gelao +gis I L North Giziga +git I L Gitxsan +giu I L Mulao +giw I L White Gelao +gix I L Gilima +giy I L Giyug +giz I L South Giziga +gji I L Geji +gjk I L Kachi Koli +gjm I E Gunditjmara +gjn I L Gonja +gju I L Gujari +gka I L Guya +gke I L Ndai +gkn I L Gokana +gko I E Kok-Nar +gkp I L Guinea Kpelle +gla gla gla gd I L Scottish Gaelic +glc I L Bon Gula +gld I L Nanai +gle gle gle ga I L Irish +glg glg glg gl I L Galician +glh I L Northwest Pashayi +gli I E Guliguli +glj I L Gula Iro +glk I L Gilaki +gll I E Garlali +glo I L Galambu +glr I L Glaro-Twabo +glu I L Gula (Chad) +glv glv glv gv I L Manx +glw I L Glavda +gly I E Gule +gma I E Gambera +gmb I L Gula'alaa +gmd I L Mághdì +gmh gmh gmh I H Middle High German (ca. 1050-1500) +gml I H Middle Low German +gmm I L Gbaya-Mbodomo +gmn I L Gimnime +gmu I L Gumalu +gmv I L Gamo +gmx I L Magoma +gmy I A Mycenaean Greek +gmz I L Mgbolizhia +gna I L Kaansa +gnb I L Gangte +gnc I E Guanche +gnd I L Zulgo-Gemzek +gne I L Ganang +gng I L Ngangam +gnh I L Lere +gni I L Gooniyandi +gnk I L //Gana +gnl I E Gangulu +gnm I L Ginuman +gnn I L Gumatj +gno I L Northern Gondi +gnq I L Gana +gnr I E Gureng Gureng +gnt I L Guntai +gnu I L Gnau +gnw I L Western Bolivian Guaraní +gnz I L Ganzi +goa I L Guro +gob I L Playero +goc I L Gorakor +god I L Godié +goe I L Gongduk +gof I L Gofa +gog I L Gogo +goh goh goh I H Old High German (ca. 750-1050) +goi I L Gobasi +goj I L Gowlan +gok I L Gowli +gol I L Gola +gom I L Goan Konkani +gon gon gon M L Gondi +goo I L Gone Dau +gop I L Yeretuar +goq I L Gorap +gor gor gor I L Gorontalo +gos I L Gronings +got got got I A Gothic +gou I L Gavar +gow I L Gorowa +gox I L Gobu +goy I L Goundo +goz I L Gozarkhani +gpa I L Gupa-Abawa +gpe I L Ghanaian Pidgin English +gpn I L Taiap +gqa I L Ga'anda +gqi I L Guiqiong +gqn I E Guana (Brazil) +gqr I L Gor +gqu I L Qau +gra I L Rajput Garasia +grb grb grb M L Grebo +grc grc grc I H Ancient Greek (to 1453) +grd I L Guruntum-Mbaaru +grg I L Madi +grh I L Gbiri-Niragu +gri I L Ghari +grj I L Southern Grebo +grm I L Kota Marudu Talantang +grn grn grn gn M L Guarani +gro I L Groma +grq I L Gorovu +grr I L Taznatit +grs I L Gresi +grt I L Garo +gru I L Kistane +grv I L Central Grebo +grw I L Gweda +grx I L Guriaso +gry I L Barclayville Grebo +grz I L Guramalum +gse I L Ghanaian Sign Language +gsg I L German Sign Language +gsl I L Gusilay +gsm I L Guatemalan Sign Language +gsn I L Gusan +gso I L Southwest Gbaya +gsp I L Wasembo +gss I L Greek Sign Language +gsw gsw gsw I L Swiss German +gta I L Guató +gti I L Gbati-ri +gtu I E Aghu-Tharnggala +gua I L Shiki +gub I L Guajajára +guc I L Wayuu +gud I L Yocoboué Dida +gue I L Gurinji +guf I L Gupapuyngu +gug I L Paraguayan Guaraní +guh I L Guahibo +gui I L Eastern Bolivian Guaraní +guj guj guj gu I L Gujarati +guk I L Gumuz +gul I L Sea Island Creole English +gum I L Guambiano +gun I L Mbyá Guaraní +guo I L Guayabero +gup I L Gunwinggu +guq I L Aché +gur I L Farefare +gus I L Guinean Sign Language +gut I L Maléku Jaíka +guu I L Yanomamö +guv I E Gey +guw I L Gun +gux I L Gourmanchéma +guz I L Gusii +gva I L Guana (Paraguay) +gvc I L Guanano +gve I L Duwet +gvf I L Golin +gvj I L Guajá +gvl I L Gulay +gvm I L Gurmana +gvn I L Kuku-Yalanji +gvo I L Gavião Do Jiparaná +gvp I L Pará Gavião +gvr I L Western Gurung +gvs I L Gumawana +gvy I E Guyani +gwa I L Mbato +gwb I L Gwa +gwc I L Kalami +gwd I L Gawwada +gwe I L Gweno +gwf I L Gowro +gwg I L Moo +gwi gwi gwi I L Gwichʼin +gwj I L /Gwi +gwm I E Awngthim +gwn I L Gwandara +gwr I L Gwere +gwt I L Gawar-Bati +gwu I E Guwamu +gww I L Kwini +gwx I L Gua +gxx I L Wè Southern +gya I L Northwest Gbaya +gyb I L Garus +gyd I L Kayardild +gye I L Gyem +gyf I E Gungabula +gyg I L Gbayi +gyi I L Gyele +gyl I L Gayil +gym I L Ngäbere +gyn I L Guyanese Creole English +gyr I L Guarayu +gyy I E Gunya +gza I L Ganza +gzi I L Gazi +gzn I L Gane +haa I L Han +hab I L Hanoi Sign Language +hac I L Gurani +had I L Hatam +hae I L Eastern Oromo +haf I L Haiphong Sign Language +hag I L Hanga +hah I L Hahon +hai hai hai M L Haida +haj I L Hajong +hak I L Hakka Chinese +hal I L Halang +ham I L Hewa +han I L Hangaza +hao I L Hakö +hap I L Hupla +haq I L Ha +har I L Harari +has I L Haisla +hat hat hat ht I L Haitian +hau hau hau ha I L Hausa +hav I L Havu +haw haw haw I L Hawaiian +hax I L Southern Haida +hay I L Haya +haz I L Hazaragi +hba I L Hamba +hbb I L Huba +hbn I L Heiban +hbo I H Ancient Hebrew +hbs sh M L Serbo-Croatian Code element for 639-1 has been deprecated +hbu I L Habu +hca I L Andaman Creole Hindi +hch I L Huichol +hdn I L Northern Haida +hds I L Honduras Sign Language +hdy I L Hadiyya +hea I L Northern Qiandong Miao +heb heb heb he I L Hebrew +hed I L Herdé +heg I L Helong +heh I L Hehe +hei I L Heiltsuk +hem I L Hemba +her her her hz I L Herero +hgm I L Hai//om +hgw I L Haigwai +hhi I L Hoia Hoia +hhr I L Kerak +hhy I L Hoyahoya +hia I L Lamang +hib I E Hibito +hid I L Hidatsa +hif I L Fiji Hindi +hig I L Kamwe +hih I L Pamosu +hii I L Hinduri +hij I L Hijuk +hik I L Seit-Kaitetu +hil hil hil I L Hiligaynon +hin hin hin hi I L Hindi +hio I L Tsoa +hir I L Himarimã +hit hit hit I A Hittite +hiw I L Hiw +hix I L Hixkaryána +hji I L Haji +hka I L Kahe +hke I L Hunde +hkk I L Hunjara-Kaina Ke +hks I L Hong Kong Sign Language +hla I L Halia +hlb I L Halbi +hld I L Halang Doan +hle I L Hlersu +hlt I L Matu Chin +hlu I A Hieroglyphic Luwian +hma I L Southern Mashan Hmong +hmb I L Humburi Senni Songhay +hmc I L Central Huishui Hmong +hmd I L Large Flowery Miao +hme I L Eastern Huishui Hmong +hmf I L Hmong Don +hmg I L Southwestern Guiyang Hmong +hmh I L Southwestern Huishui Hmong +hmi I L Northern Huishui Hmong +hmj I L Ge +hmk I E Maek +hml I L Luopohe Hmong +hmm I L Central Mashan Hmong +hmn hmn hmn M L Hmong +hmo hmo hmo ho I L Hiri Motu +hmp I L Northern Mashan Hmong +hmq I L Eastern Qiandong Miao +hmr I L Hmar +hms I L Southern Qiandong Miao +hmt I L Hamtai +hmu I L Hamap +hmv I L Hmong Dô +hmw I L Western Mashan Hmong +hmy I L Southern Guiyang Hmong +hmz I L Hmong Shua +hna I L Mina (Cameroon) +hnd I L Southern Hindko +hne I L Chhattisgarhi +hnh I L //Ani +hni I L Hani +hnj I L Hmong Njua +hnn I L Hanunoo +hno I L Northern Hindko +hns I L Caribbean Hindustani +hnu I L Hung +hoa I L Hoava +hob I L Mari (Madang Province) +hoc I L Ho +hod I E Holma +hoe I L Horom +hoh I L Hobyót +hoi I L Holikachuk +hoj I L Hadothi +hol I L Holu +hom I E Homa +hoo I L Holoholo +hop I L Hopi +hor I E Horo +hos I L Ho Chi Minh City Sign Language +hot I L Hote +hov I L Hovongan +how I L Honi +hoy I L Holiya +hoz I L Hozo +hpo I L Hpon +hps I L Hawai'i Pidgin Sign Language +hra I L Hrangkhol +hrc I L Niwer Mil +hre I L Hre +hrk I L Haruku +hrm I L Horned Miao +hro I L Haroi +hrp I E Nhirrpi +hrt I L Hértevin +hru I L Hruso +hrv hrv hrv hr I L Croatian +hrw I L Warwar Feni +hrx I L Hunsrik +hrz I L Harzani +hsb hsb hsb I L Upper Sorbian +hsh I L Hungarian Sign Language +hsl I L Hausa Sign Language +hsn I L Xiang Chinese +hss I L Harsusi +hti I L Hoti +hto I L Minica Huitoto +hts I L Hadza +htu I L Hitu +htx I A Middle Hittite +hub I L Huambisa +huc I L =/Hua +hud I L Huaulu +hue I L San Francisco Del Mar Huave +huf I L Humene +hug I L Huachipaeri +huh I L Huilliche +hui I L Huli +huj I L Northern Guiyang Hmong +huk I L Hulung +hul I L Hula +hum I L Hungana +hun hun hun hu I L Hungarian +huo I L Hu +hup hup hup I L Hupa +huq I L Tsat +hur I L Halkomelem +hus I L Huastec +hut I L Humla +huu I L Murui Huitoto +huv I L San Mateo Del Mar Huave +huw I E Hukumina +hux I L Nüpode Huitoto +huy I L Hulaulá +huz I L Hunzib +hvc I L Haitian Vodoun Culture Language +hve I L San Dionisio Del Mar Huave +hvk I L Haveke +hvn I L Sabu +hvv I L Santa María Del Mar Huave +hwa I L Wané +hwc I L Hawai'i Creole English +hwo I L Hwana +hya I L Hya +hye arm hye hy I L Armenian +iai I L Iaai +ian I L Iatmul +iap I L Iapama +iar I L Purari +iba iba iba I L Iban +ibb I L Ibibio +ibd I L Iwaidja +ibe I L Akpes +ibg I L Ibanag +ibl I L Ibaloi +ibm I L Agoi +ibn I L Ibino +ibo ibo ibo ig I L Igbo +ibr I L Ibuoro +ibu I L Ibu +iby I L Ibani +ica I L Ede Ica +ich I L Etkywan +icl I L Icelandic Sign Language +icr I L Islander Creole English +ida I L Idakho-Isukha-Tiriki +idb I L Indo-Portuguese +idc I L Idon +idd I L Ede Idaca +ide I L Idere +idi I L Idi +ido ido ido io I C Ido +idr I L Indri +ids I L Idesa +idt I L Idaté +idu I L Idoma +ifa I L Amganad Ifugao +ifb I L Batad Ifugao +ife I L Ifè +iff I E Ifo +ifk I L Tuwali Ifugao +ifm I L Teke-Fuumu +ifu I L Mayoyao Ifugao +ify I L Keley-I Kallahan +igb I L Ebira +ige I L Igede +igg I L Igana +igl I L Igala +igm I L Kanggape +ign I L Ignaciano +igo I L Isebe +igs I C Interglossa +igw I L Igwe +ihb I L Iha Based Pidgin +ihi I L Ihievbe +ihp I L Iha +ihw I E Bidhawal +iii iii iii ii I L Sichuan Yi +iin I E Thiin +ijc I L Izon +ije I L Biseni +ijj I L Ede Ije +ijn I L Kalabari +ijs I L Southeast Ijo +ike I L Eastern Canadian Inuktitut +iki I L Iko +ikk I L Ika +ikl I L Ikulu +iko I L Olulumo-Ikom +ikp I L Ikpeshi +ikr I E Ikaranggal +ikt I L Inuinnaqtun +iku iku iku iu M L Inuktitut +ikv I L Iku-Gora-Ankwa +ikw I L Ikwere +ikx I L Ik +ikz I L Ikizu +ila I L Ile Ape +ilb I L Ila +ile ile ile ie I C Interlingue +ilg I E Garig-Ilgar +ili I L Ili Turki +ilk I L Ilongot +ill I L Iranun +ilo ilo ilo I L Iloko +ils I L International Sign +ilu I L Ili'uun +ilv I L Ilue +ima I L Mala Malasar +ime I L Imeraguen +imi I L Anamgura +iml I E Miluk +imn I L Imonda +imo I L Imbongu +imr I L Imroing +ims I A Marsian +imy I A Milyan +ina ina ina ia I C Interlingua (International Auxiliary Language Association) +inb I L Inga +ind ind ind id I L Indonesian +ing I L Degexit'an +inh inh inh I L Ingush +inj I L Jungle Inga +inl I L Indonesian Sign Language +inm I A Minaean +inn I L Isinai +ino I L Inoke-Yate +inp I L Iñapari +ins I L Indian Sign Language +int I L Intha +inz I E Ineseño +ior I L Inor +iou I L Tuma-Irumu +iow I E Iowa-Oto +ipi I L Ipili +ipk ipk ipk ik M L Inupiaq +ipo I L Ipiko +iqu I L Iquito +iqw I L Ikwo +ire I L Iresim +irh I L Irarutu +iri I L Irigwe +irk I L Iraqw +irn I L Irántxe +irr I L Ir +iru I L Irula +irx I L Kamberau +iry I L Iraya +isa I L Isabi +isc I L Isconahua +isd I L Isnag +ise I L Italian Sign Language +isg I L Irish Sign Language +ish I L Esan +isi I L Nkem-Nkum +isk I L Ishkashimi +isl ice isl is I L Icelandic +ism I L Masimasi +isn I L Isanzu +iso I L Isoko +isr I L Israeli Sign Language +ist I L Istriot +isu I L Isu (Menchum Division) +ita ita ita it I L Italian +itb I L Binongan Itneg +ite I E Itene +iti I L Inlaod Itneg +itk I L Judeo-Italian +itl I L Itelmen +itm I L Itu Mbon Uzo +ito I L Itonama +itr I L Iteri +its I L Isekiri +itt I L Maeng Itneg +itv I L Itawit +itw I L Ito +itx I L Itik +ity I L Moyadan Itneg +itz I L Itzá +ium I L Iu Mien +ivb I L Ibatan +ivv I L Ivatan +iwk I L I-Wak +iwm I L Iwam +iwo I L Iwur +iws I L Sepik Iwam +ixc I L Ixcatec +ixl I L Ixil +iya I L Iyayu +iyo I L Mesaka +iyx I L Yaka (Congo) +izh I L Ingrian +izr I L Izere +izz I L Izii +jaa I L Jamamadí +jab I L Hyam +jac I L Popti' +jad I L Jahanka +jae I L Yabem +jaf I L Jara +jah I L Jah Hut +jaj I L Zazao +jak I L Jakun +jal I L Yalahatan +jam I L Jamaican Creole English +jan I E Jandai +jao I L Yanyuwa +jaq I L Yaqay +jas I L New Caledonian Javanese +jat I L Jakati +jau I L Yaur +jav jav jav jv I L Javanese +jax I L Jambi Malay +jay I L Yan-nhangu +jaz I L Jawe +jbe I L Judeo-Berber +jbi I E Badjiri +jbj I L Arandai +jbk I L Barikewa +jbn I L Nafusi +jbo jbo jbo I C Lojban +jbr I L Jofotek-Bromnya +jbt I L Jabutí +jbu I L Jukun Takum +jbw I E Yawijibaya +jcs I L Jamaican Country Sign Language +jct I L Krymchak +jda I L Jad +jdg I L Jadgali +jdt I L Judeo-Tat +jeb I L Jebero +jee I L Jerung +jeg I L Jeng +jeh I L Jeh +jei I L Yei +jek I L Jeri Kuo +jel I L Yelmek +jen I L Dza +jer I L Jere +jet I L Manem +jeu I L Jonkor Bourmataguil +jgb I E Ngbee +jge I L Judeo-Georgian +jgk I L Gwak +jgo I L Ngomba +jhi I L Jehai +jhs I L Jhankot Sign Language +jia I L Jina +jib I L Jibu +jic I L Tol +jid I L Bu +jie I L Jilbe +jig I L Djingili +jih I L sTodsde +jii I L Jiiddu +jil I L Jilim +jim I L Jimi (Cameroon) +jio I L Jiamao +jiq I L Guanyinqiao +jit I L Jita +jiu I L Youle Jinuo +jiv I L Shuar +jiy I L Buyuan Jinuo +jjr I L Bankal +jkm I L Mobwa Karen +jko I L Kubo +jkp I L Paku Karen +jkr I L Koro (India) +jku I L Labir +jle I L Ngile +jls I L Jamaican Sign Language +jma I L Dima +jmb I L Zumbun +jmc I L Machame +jmd I L Yamdena +jmi I L Jimi (Nigeria) +jml I L Jumli +jmn I L Makuri Naga +jmr I L Kamara +jms I L Mashi (Nigeria) +jmw I L Mouwase +jmx I L Western Juxtlahuaca Mixtec +jna I L Jangshung +jnd I L Jandavra +jng I E Yangman +jni I L Janji +jnj I L Yemsa +jnl I L Rawat +jns I L Jaunsari +job I L Joba +jod I L Wojenaka +jor I E Jorá +jos I L Jordanian Sign Language +jow I L Jowulu +jpa I H Jewish Palestinian Aramaic +jpn jpn jpn ja I L Japanese +jpr jpr jpr I L Judeo-Persian +jqr I L Jaqaru +jra I L Jarai +jrb jrb jrb M L Judeo-Arabic +jrr I L Jiru +jrt I L Jorto +jru I L Japrería +jsl I L Japanese Sign Language +jua I L Júma +jub I L Wannu +juc I E Jurchen +jud I L Worodougou +juh I L Hõne +jui I E Ngadjuri +juk I L Wapan +jul I L Jirel +jum I L Jumjum +jun I L Juang +juo I L Jiba +jup I L Hupdë +jur I L Jurúna +jus I L Jumla Sign Language +jut I L Jutish +juu I L Ju +juw I L Wãpha +juy I L Juray +jvd I E Javindo +jvn I L Caribbean Javanese +jwi I L Jwira-Pepesa +jya I L Jiarong +jye I L Judeo-Yemeni Arabic +jyy I L Jaya +kaa kaa kaa I L Kara-Kalpak +kab kab kab I L Kabyle +kac kac kac I L Kachin +kad I L Adara +kae I E Ketangalan +kaf I L Katso +kag I L Kajaman +kah I L Kara (Central African Republic) +kai I L Karekare +kaj I L Jju +kak I L Kayapa Kallahan +kal kal kal kl I L Kalaallisut +kam kam kam I L Kamba (Kenya) +kan kan kan kn I L Kannada +kao I L Xaasongaxango +kap I L Bezhta +kaq I L Capanahua +kas kas kas ks I L Kashmiri +kat geo kat ka I L Georgian +kau kau kau kr M L Kanuri +kav I L Katukína +kaw kaw kaw I A Kawi +kax I L Kao +kay I L Kamayurá +kaz kaz kaz kk I L Kazakh +kba I E Kalarko +kbb I E Kaxuiâna +kbc I L Kadiwéu +kbd kbd kbd I L Kabardian +kbe I L Kanju +kbf I E Kakauhua +kbg I L Khamba +kbh I L Camsá +kbi I L Kaptiau +kbj I L Kari +kbk I L Grass Koiari +kbl I L Kanembu +kbm I L Iwal +kbn I L Kare (Central African Republic) +kbo I L Keliko +kbp I L Kabiyè +kbq I L Kamano +kbr I L Kafa +kbs I L Kande +kbt I L Abadi +kbu I L Kabutra +kbv I L Dera (Indonesia) +kbw I L Kaiep +kbx I L Ap Ma +kby I L Manga Kanuri +kbz I L Duhwa +kca I L Khanty +kcb I L Kawacha +kcc I L Lubila +kcd I L Ngkâlmpw Kanum +kce I L Kaivi +kcf I L Ukaan +kcg I L Tyap +kch I L Vono +kci I L Kamantan +kcj I L Kobiana +kck I L Kalanga +kcl I L Kela (Papua New Guinea) +kcm I L Gula (Central African Republic) +kcn I L Nubi +kco I L Kinalakna +kcp I L Kanga +kcq I L Kamo +kcr I L Katla +kcs I L Koenoem +kct I L Kaian +kcu I L Kami (Tanzania) +kcv I L Kete +kcw I L Kabwari +kcx I L Kachama-Ganjule +kcy I L Korandje +kcz I L Konongo +kda I E Worimi +kdc I L Kutu +kdd I L Yankunytjatjara +kde I L Makonde +kdf I L Mamusi +kdg I L Seba +kdh I L Tem +kdi I L Kumam +kdj I L Karamojong +kdk I L Numèè +kdl I L Tsikimba +kdm I L Kagoma +kdn I L Kunda +kdp I L Kaningdon-Nindem +kdq I L Koch +kdr I L Karaim +kdt I L Kuy +kdu I L Kadaru +kdw I L Koneraw +kdx I L Kam +kdy I L Keder +kdz I L Kwaja +kea I L Kabuverdianu +keb I L Kélé +kec I L Keiga +ked I L Kerewe +kee I L Eastern Keres +kef I L Kpessi +keg I L Tese +keh I L Keak +kei I L Kei +kej I L Kadar +kek I L Kekchí +kel I L Kela (Democratic Republic of Congo) +kem I L Kemak +ken I L Kenyang +keo I L Kakwa +kep I L Kaikadi +keq I L Kamar +ker I L Kera +kes I L Kugbo +ket I L Ket +keu I L Akebu +kev I L Kanikkaran +kew I L West Kewa +kex I L Kukna +key I L Kupia +kez I L Kukele +kfa I L Kodava +kfb I L Northwestern Kolami +kfc I L Konda-Dora +kfd I L Korra Koraga +kfe I L Kota (India) +kff I L Koya +kfg I L Kudiya +kfh I L Kurichiya +kfi I L Kannada Kurumba +kfj I L Kemiehua +kfk I L Kinnauri +kfl I L Kung +kfm I L Khunsari +kfn I L Kuk +kfo I L Koro (Côte d'Ivoire) +kfp I L Korwa +kfq I L Korku +kfr I L Kachchi +kfs I L Bilaspuri +kft I L Kanjari +kfu I L Katkari +kfv I L Kurmukar +kfw I L Kharam Naga +kfx I L Kullu Pahari +kfy I L Kumaoni +kfz I L Koromfé +kga I L Koyaga +kgb I L Kawe +kgc I L Kasseng +kgd I L Kataang +kge I L Komering +kgf I L Kube +kgg I L Kusunda +kgi I L Selangor Sign Language +kgj I L Gamale Kham +kgk I L Kaiwá +kgl I E Kunggari +kgm I E Karipúna +kgn I L Karingani +kgo I L Krongo +kgp I L Kaingang +kgq I L Kamoro +kgr I L Abun +kgs I L Kumbainggar +kgt I L Somyev +kgu I L Kobol +kgv I L Karas +kgw I L Karon Dori +kgx I L Kamaru +kgy I L Kyerung +kha kha kha I L Khasi +khb I L Lü +khc I L Tukang Besi North +khd I L Bädi Kanum +khe I L Korowai +khf I L Khuen +khg I L Khams Tibetan +khh I L Kehu +khj I L Kuturmi +khk I L Halh Mongolian +khl I L Lusi +khm khm khm km I L Central Khmer +khn I L Khandesi +kho kho kho I A Khotanese +khp I L Kapori +khq I L Koyra Chiini Songhay +khr I L Kharia +khs I L Kasua +kht I L Khamti +khu I L Nkhumbi +khv I L Khvarshi +khw I L Khowar +khx I L Kanu +khy I L Kele (Democratic Republic of Congo) +khz I L Keapara +kia I L Kim +kib I L Koalib +kic I L Kickapoo +kid I L Koshin +kie I L Kibet +kif I L Eastern Parbate Kham +kig I L Kimaama +kih I L Kilmeri +kii I E Kitsai +kij I L Kilivila +kik kik kik ki I L Kikuyu +kil I L Kariya +kim I L Karagas +kin kin kin rw I L Kinyarwanda +kio I L Kiowa +kip I L Sheshi Kham +kiq I L Kosadle +kir kir kir ky I L Kirghiz +kis I L Kis +kit I L Agob +kiu I L Kirmanjki (individual language) +kiv I L Kimbu +kiw I L Northeast Kiwai +kix I L Khiamniungan Naga +kiy I L Kirikiri +kiz I L Kisi +kja I L Mlap +kjb I L Q'anjob'al +kjc I L Coastal Konjo +kjd I L Southern Kiwai +kje I L Kisar +kjf I L Khalaj +kjg I L Khmu +kjh I L Khakas +kji I L Zabana +kjj I L Khinalugh +kjk I L Highland Konjo +kjl I L Western Parbate Kham +kjm I L Kháng +kjn I L Kunjen +kjo I L Harijan Kinnauri +kjp I L Pwo Eastern Karen +kjq I L Western Keres +kjr I L Kurudu +kjs I L East Kewa +kjt I L Phrae Pwo Karen +kju I L Kashaya +kjx I L Ramopa +kjy I L Erave +kjz I L Bumthangkha +kka I L Kakanda +kkb I L Kwerisa +kkc I L Odoodee +kkd I L Kinuku +kke I L Kakabe +kkf I L Kalaktang Monpa +kkg I L Mabaka Valley Kalinga +kkh I L Khün +kki I L Kagulu +kkj I L Kako +kkk I L Kokota +kkl I L Kosarek Yale +kkm I L Kiong +kkn I L Kon Keu +kko I L Karko +kkp I L Gugubera +kkq I L Kaiku +kkr I L Kir-Balar +kks I L Giiwo +kkt I L Koi +kku I L Tumi +kkv I L Kangean +kkw I L Teke-Kukuya +kkx I L Kohin +kky I L Guguyimidjir +kkz I L Kaska +kla I E Klamath-Modoc +klb I L Kiliwa +klc I L Kolbila +kld I L Gamilaraay +kle I L Kulung (Nepal) +klf I L Kendeje +klg I L Tagakaulo +klh I L Weliki +kli I L Kalumpang +klj I L Turkic Khalaj +klk I L Kono (Nigeria) +kll I L Kagan Kalagan +klm I L Migum +kln M L Kalenjin +klo I L Kapya +klp I L Kamasa +klq I L Rumu +klr I L Khaling +kls I L Kalasha +klt I L Nukna +klu I L Klao +klv I L Maskelynes +klw I L Lindu +klx I L Koluwawa +kly I L Kalao +klz I L Kabola +kma I L Konni +kmb kmb kmb I L Kimbundu +kmc I L Southern Dong +kmd I L Majukayang Kalinga +kme I L Bakole +kmf I L Kare (Papua New Guinea) +kmg I L Kâte +kmh I L Kalam +kmi I L Kami (Nigeria) +kmj I L Kumarbhag Paharia +kmk I L Limos Kalinga +kml I L Tanudan Kalinga +kmm I L Kom (India) +kmn I L Awtuw +kmo I L Kwoma +kmp I L Gimme +kmq I L Kwama +kmr I L Northern Kurdish +kms I L Kamasau +kmt I L Kemtuik +kmu I L Kanite +kmv I L Karipúna Creole French +kmw I L Komo (Democratic Republic of Congo) +kmx I L Waboda +kmy I L Koma +kmz I L Khorasani Turkish +kna I L Dera (Nigeria) +knb I L Lubuagan Kalinga +knc I L Central Kanuri +knd I L Konda +kne I L Kankanaey +knf I L Mankanya +kng I L Koongo +kni I L Kanufi +knj I L Western Kanjobal +knk I L Kuranko +knl I L Keninjal +knm I L Kanamarí +knn I L Konkani (individual language) +kno I L Kono (Sierra Leone) +knp I L Kwanja +knq I L Kintaq +knr I L Kaningra +kns I L Kensiu +knt I L Panoan Katukína +knu I L Kono (Guinea) +knv I L Tabo +knw I L Kung-Ekoka +knx I L Kendayan +kny I L Kanyok +knz I L Kalamsé +koa I L Konomala +koc I E Kpati +kod I L Kodi +koe I L Kacipo-Balesi +kof I E Kubi +kog I L Cogui +koh I L Koyo +koi I L Komi-Permyak +koj I L Sara Dunjo +kok kok kok M L Konkani (macrolanguage) +kol I L Kol (Papua New Guinea) +kom kom kom kv M L Komi +kon kon kon kg M L Kongo +koo I L Konzo +kop I L Waube +koq I L Kota (Gabon) +kor kor kor ko I L Korean +kos kos kos I L Kosraean +kot I L Lagwan +kou I L Koke +kov I L Kudu-Camo +kow I L Kugama +kox I E Coxima +koy I L Koyukon +koz I L Korak +kpa I L Kutto +kpb I L Mullu Kurumba +kpc I L Curripaco +kpd I L Koba +kpe kpe kpe M L Kpelle +kpf I L Komba +kpg I L Kapingamarangi +kph I L Kplang +kpi I L Kofei +kpj I L Karajá +kpk I L Kpan +kpl I L Kpala +kpm I L Koho +kpn I E Kepkiriwát +kpo I L Ikposo +kpq I L Korupun-Sela +kpr I L Korafe-Yegha +kps I L Tehit +kpt I L Karata +kpu I L Kafoa +kpv I L Komi-Zyrian +kpw I L Kobon +kpx I L Mountain Koiali +kpy I L Koryak +kpz I L Kupsabiny +kqa I L Mum +kqb I L Kovai +kqc I L Doromu-Koki +kqd I L Koy Sanjaq Surat +kqe I L Kalagan +kqf I L Kakabai +kqg I L Khe +kqh I L Kisankasa +kqi I L Koitabu +kqj I L Koromira +kqk I L Kotafon Gbe +kql I L Kyenele +kqm I L Khisa +kqn I L Kaonde +kqo I L Eastern Krahn +kqp I L Kimré +kqq I L Krenak +kqr I L Kimaragang +kqs I L Northern Kissi +kqt I L Klias River Kadazan +kqu I E Seroa +kqv I L Okolod +kqw I L Kandas +kqx I L Mser +kqy I L Koorete +kqz I E Korana +kra I L Kumhali +krb I E Karkin +krc krc krc I L Karachay-Balkar +krd I L Kairui-Midiki +kre I L Panará +krf I L Koro (Vanuatu) +krh I L Kurama +kri I L Krio +krj I L Kinaray-A +krk I E Kerek +krl krl krl I L Karelian +krm I L Krim +krn I L Sapo +krp I L Korop +krr I L Kru'ng 2 +krs I L Gbaya (Sudan) +krt I L Tumari Kanuri +kru kru kru I L Kurukh +krv I L Kavet +krw I L Western Krahn +krx I L Karon +kry I L Kryts +krz I L Sota Kanum +ksa I L Shuwa-Zamani +ksb I L Shambala +ksc I L Southern Kalinga +ksd I L Kuanua +kse I L Kuni +ksf I L Bafia +ksg I L Kusaghe +ksh I L Kölsch +ksi I L Krisa +ksj I L Uare +ksk I L Kansa +ksl I L Kumalu +ksm I L Kumba +ksn I L Kasiguranin +kso I L Kofa +ksp I L Kaba +ksq I L Kwaami +ksr I L Borong +kss I L Southern Kisi +kst I L Winyé +ksu I L Khamyang +ksv I L Kusu +ksw I L S'gaw Karen +ksx I L Kedang +ksy I L Kharia Thar +ksz I L Kodaku +kta I L Katua +ktb I L Kambaata +ktc I L Kholok +ktd I L Kokata +kte I L Nubri +ktf I L Kwami +ktg I E Kalkutung +kth I L Karanga +kti I L North Muyu +ktj I L Plapo Krumen +ktk I E Kaniet +ktl I L Koroshi +ktm I L Kurti +ktn I L Karitiâna +kto I L Kuot +ktp I L Kaduo +ktq I E Katabaga +ktr I L Kota Marudu Tinagas +kts I L South Muyu +ktt I L Ketum +ktu I L Kituba (Democratic Republic of Congo) +ktv I L Eastern Katu +ktw I E Kato +ktx I L Kaxararí +kty I L Kango (Bas-Uélé District) +ktz I L Ju/'hoan +kua kua kua kj I L Kuanyama +kub I L Kutep +kuc I L Kwinsu +kud I L 'Auhelawa +kue I L Kuman +kuf I L Western Katu +kug I L Kupa +kuh I L Kushi +kui I L Kuikúro-Kalapálo +kuj I L Kuria +kuk I L Kepo' +kul I L Kulere +kum kum kum I L Kumyk +kun I L Kunama +kuo I L Kumukio +kup I L Kunimaipa +kuq I L Karipuna +kur kur kur ku M L Kurdish +kus I L Kusaal +kut kut kut I L Kutenai +kuu I L Upper Kuskokwim +kuv I L Kur +kuw I L Kpagua +kux I L Kukatja +kuy I L Kuuku-Ya'u +kuz I E Kunza +kva I L Bagvalal +kvb I L Kubu +kvc I L Kove +kvd I L Kui (Indonesia) +kve I L Kalabakan +kvf I L Kabalai +kvg I L Kuni-Boazi +kvh I L Komodo +kvi I L Kwang +kvj I L Psikye +kvk I L Korean Sign Language +kvl I L Kayaw +kvm I L Kendem +kvn I L Border Kuna +kvo I L Dobel +kvp I L Kompane +kvq I L Geba Karen +kvr I L Kerinci +kvs I L Kunggara +kvt I L Lahta Karen +kvu I L Yinbaw Karen +kvv I L Kola +kvw I L Wersing +kvx I L Parkari Koli +kvy I L Yintale Karen +kvz I L Tsakwambo +kwa I L Dâw +kwb I L Kwa +kwc I L Likwala +kwd I L Kwaio +kwe I L Kwerba +kwf I L Kwara'ae +kwg I L Sara Kaba Deme +kwh I L Kowiai +kwi I L Awa-Cuaiquer +kwj I L Kwanga +kwk I L Kwakiutl +kwl I L Kofyar +kwm I L Kwambi +kwn I L Kwangali +kwo I L Kwomtari +kwp I L Kodia +kwq I L Kwak +kwr I L Kwer +kws I L Kwese +kwt I L Kwesten +kwu I L Kwakum +kwv I L Sara Kaba Náà +kww I L Kwinti +kwx I L Khirwar +kwy I L San Salvador Kongo +kwz I E Kwadi +kxa I L Kairiru +kxb I L Krobu +kxc I L Konso +kxd I L Brunei +kxe I L Kakihum +kxf I L Manumanaw Karen +kxh I L Karo (Ethiopia) +kxi I L Keningau Murut +kxj I L Kulfa +kxk I L Zayein Karen +kxl I L Nepali Kurux +kxm I L Northern Khmer +kxn I L Kanowit-Tanjong Melanau +kxo I E Kanoé +kxp I L Wadiyara Koli +kxq I L Smärky Kanum +kxr I L Koro (Papua New Guinea) +kxs I L Kangjia +kxt I L Koiwat +kxu I L Kui (India) +kxv I L Kuvi +kxw I L Konai +kxx I L Likuba +kxy I L Kayong +kxz I L Kerewo +kya I L Kwaya +kyb I L Butbut Kalinga +kyc I L Kyaka +kyd I L Karey +kye I L Krache +kyf I L Kouya +kyg I L Keyagana +kyh I L Karok +kyi I L Kiput +kyj I L Karao +kyk I L Kamayo +kyl I L Kalapuya +kym I L Kpatili +kyn I L Northern Binukidnon +kyo I L Kelon +kyp I L Kang +kyq I L Kenga +kyr I L Kuruáya +kys I L Baram Kayan +kyt I L Kayagar +kyu I L Western Kayah +kyv I L Kayort +kyw I L Kudmali +kyx I L Rapoisi +kyy I L Kambaira +kyz I L Kayabí +kza I L Western Karaboro +kzb I L Kaibobo +kzc I L Bondoukou Kulango +kzd I L Kadai +kze I L Kosena +kzf I L Da'a Kaili +kzg I L Kikai +kzi I L Kelabit +kzj I L Coastal Kadazan +kzk I E Kazukuru +kzl I L Kayeli +kzm I L Kais +kzn I L Kokola +kzo I L Kaningi +kzp I L Kaidipang +kzq I L Kaike +kzr I L Karang +kzs I L Sugut Dusun +kzt I L Tambunan Dusun +kzu I L Kayupulau +kzv I L Komyandaret +kzw I E Karirí-Xocó +kzx I L Kamarian +kzy I L Kango (Tshopo District) +kzz I L Kalabra +laa I L Southern Subanen +lab I A Linear A +lac I L Lacandon +lad lad lad I L Ladino +lae I L Pattani +laf I L Lafofa +lag I L Langi +lah lah lah M L Lahnda +lai I L Lambya +laj I L Lango (Uganda) +lak I L Laka (Nigeria) +lal I L Lalia +lam lam lam I L Lamba +lan I L Laru +lao lao lao lo I L Lao +lap I L Laka (Chad) +laq I L Qabiao +lar I L Larteh +las I L Lama (Togo) +lat lat lat la I A Latin +lau I L Laba +lav lav lav lv M L Latvian +law I L Lauje +lax I L Tiwa +lay I L Lama (Myanmar) +laz I E Aribwatsa +lba I E Lui +lbb I L Label +lbc I L Lakkia +lbe I L Lak +lbf I L Tinani +lbg I L Laopang +lbi I L La'bi +lbj I L Ladakhi +lbk I L Central Bontok +lbl I L Libon Bikol +lbm I L Lodhi +lbn I L Lamet +lbo I L Laven +lbq I L Wampar +lbr I L Lohorung +lbs I L Libyan Sign Language +lbt I L Lachi +lbu I L Labu +lbv I L Lavatbura-Lamusong +lbw I L Tolaki +lbx I L Lawangan +lby I E Lamu-Lamu +lbz I L Lardil +lcc I L Legenyem +lcd I L Lola +lce I L Loncong +lcf I L Lubu +lch I L Luchazi +lcl I L Lisela +lcm I L Tungag +lcp I L Western Lawa +lcq I L Luhu +lcs I L Lisabata-Nuniali +lda I L Kla-Dan +ldb I L Dũya +ldd I L Luri +ldg I L Lenyima +ldh I L Lamja-Dengsa-Tola +ldi I L Laari +ldj I L Lemoro +ldk I L Leelau +ldl I L Kaan +ldm I L Landoma +ldn I C Láadan +ldo I L Loo +ldp I L Tso +ldq I L Lufu +lea I L Lega-Shabunda +leb I L Lala-Bisa +lec I L Leco +led I L Lendu +lee I L Lyélé +lef I L Lelemi +leg I L Lengua +leh I L Lenje +lei I L Lemio +lej I L Lengola +lek I L Leipon +lel I L Lele (Democratic Republic of Congo) +lem I L Nomaande +len I E Lenca +leo I L Leti (Cameroon) +lep I L Lepcha +leq I L Lembena +ler I L Lenkau +les I L Lese +let I L Lesing-Gelimi +leu I L Kara (Papua New Guinea) +lev I L Lamma +lew I L Ledo Kaili +lex I L Luang +ley I L Lemolang +lez lez lez I L Lezghian +lfa I L Lefa +lfn I C Lingua Franca Nova +lga I L Lungga +lgb I L Laghu +lgg I L Lugbara +lgh I L Laghuu +lgi I L Lengilu +lgk I L Lingarak +lgl I L Wala +lgm I L Lega-Mwenga +lgn I L Opuuo +lgq I L Logba +lgr I L Lengo +lgt I L Pahi +lgu I L Longgu +lgz I L Ligenza +lha I L Laha (Viet Nam) +lhh I L Laha (Indonesia) +lhi I L Lahu Shi +lhl I L Lahul Lohar +lhm I L Lhomi +lhn I L Lahanan +lhp I L Lhokpu +lhs I E Mlahsö +lht I L Lo-Toga +lhu I L Lahu +lia I L West-Central Limba +lib I L Likum +lic I L Hlai +lid I L Nyindrou +lie I L Likila +lif I L Limbu +lig I L Ligbi +lih I L Lihir +lii I L Lingkhim +lij I L Ligurian +lik I L Lika +lil I L Lillooet +lim lim lim li I L Limburgan +lin lin lin ln I L Lingala +lio I L Liki +lip I L Sekpele +liq I L Libido +lir I L Liberian English +lis I L Lisu +lit lit lit lt I L Lithuanian +liu I L Logorik +liv I L Liv +liw I L Col +lix I L Liabuku +liy I L Banda-Bambari +liz I L Libinza +lja I E Golpa +lje I L Rampi +lji I L Laiyolo +ljl I L Li'o +ljp I L Lampung Api +ljw I L Yirandali +ljx I E Yuru +lka I L Lakalei +lkb I L Kabras +lkc I L Kucong +lkd I L Lakondê +lke I L Kenyi +lkh I L Lakha +lki I L Laki +lkj I L Remun +lkl I L Laeko-Libuat +lkm I E Kalaamaya +lkn I L Lakon +lko I L Khayo +lkr I L Päri +lks I L Kisa +lkt I L Lakota +lku I E Kungkari +lky I L Lokoya +lla I L Lala-Roba +llb I L Lolo +llc I L Lele (Guinea) +lld I L Ladin +lle I L Lele (Papua New Guinea) +llf I E Hermit +llg I L Lole +llh I L Lamu +lli I L Teke-Laali +llj I E Ladji Ladji +llk I E Lelak +lll I L Lilau +llm I L Lasalimu +lln I L Lele (Chad) +llo I L Khlor +llp I L North Efate +llq I L Lolak +lls I L Lithuanian Sign Language +llu I L Lau +llx I L Lauan +lma I L East Limba +lmb I L Merei +lmc I E Limilngan +lmd I L Lumun +lme I L Pévé +lmf I L South Lembata +lmg I L Lamogai +lmh I L Lambichhong +lmi I L Lombi +lmj I L West Lembata +lmk I L Lamkang +lml I L Hano +lmm I L Lamam +lmn I L Lambadi +lmo I L Lombard +lmp I L Limbum +lmq I L Lamatuka +lmr I L Lamalera +lmu I L Lamenu +lmv I L Lomaiviti +lmw I L Lake Miwok +lmx I L Laimbue +lmy I L Lamboya +lmz I E Lumbee +lna I L Langbashe +lnb I L Mbalanhu +lnd I L Lundayeh +lng I A Langobardic +lnh I L Lanoh +lni I L Daantanai' +lnj I E Leningitij +lnl I L South Central Banda +lnm I L Langam +lnn I L Lorediakarkar +lno I L Lango (Sudan) +lns I L Lamnso' +lnu I L Longuda +lnw I E Lanima +lnz I L Lonzo +loa I L Loloda +lob I L Lobi +loc I L Inonhan +loe I L Saluan +lof I L Logol +log I L Logo +loh I L Narim +loi I L Loma (Côte d'Ivoire) +loj I L Lou +lok I L Loko +lol lol lol I L Mongo +lom I L Loma (Liberia) +lon I L Malawi Lomwe +loo I L Lombo +lop I L Lopa +loq I L Lobala +lor I L Téén +los I L Loniu +lot I L Otuho +lou I L Louisiana Creole French +lov I L Lopi +low I L Tampias Lobu +lox I L Loun +loy I L Loke +loz loz loz I L Lozi +lpa I L Lelepa +lpe I L Lepki +lpn I L Long Phuri Naga +lpo I L Lipo +lpx I L Lopit +lra I L Rara Bakati' +lrc I L Northern Luri +lre I E Laurentian +lrg I E Laragia +lri I L Marachi +lrk I L Loarki +lrl I L Lari +lrm I L Marama +lrn I L Lorang +lro I L Laro +lrr I L Southern Yamphu +lrt I L Larantuka Malay +lrv I L Larevat +lrz I L Lemerig +lsa I L Lasgerdi +lsd I L Lishana Deni +lse I L Lusengo +lsg I L Lyons Sign Language +lsh I L Lish +lsi I L Lashi +lsl I L Latvian Sign Language +lsm I L Saamia +lso I L Laos Sign Language +lsp I L Panamanian Sign Language +lsr I L Aruop +lss I L Lasi +lst I L Trinidad and Tobago Sign Language +lsy I L Mauritian Sign Language +ltc I H Late Middle Chinese +ltg I L Latgalian +lti I L Leti (Indonesia) +ltn I L Latundê +lto I L Tsotso +lts I L Tachoni +ltu I L Latu +ltz ltz ltz lb I L Luxembourgish +lua lua lua I L Luba-Lulua +lub lub lub lu I L Luba-Katanga +luc I L Aringa +lud I L Ludian +lue I L Luvale +luf I L Laua +lug lug lug lg I L Ganda +lui lui lui I L Luiseno +luj I L Luna +luk I L Lunanakha +lul I L Olu'bo +lum I L Luimbi +lun lun lun I L Lunda +luo luo luo I L Luo (Kenya and Tanzania) +lup I L Lumbu +luq I L Lucumi +lur I L Laura +lus lus lus I L Lushai +lut I L Lushootseed +luu I L Lumba-Yakkha +luv I L Luwati +luw I L Luo (Cameroon) +luy M L Luyia +luz I L Southern Luri +lva I L Maku'a +lvk I L Lavukaleve +lvs I L Standard Latvian +lvu I L Levuka +lwa I L Lwalu +lwe I L Lewo Eleng +lwg I L Wanga +lwh I L White Lachi +lwl I L Eastern Lawa +lwm I L Laomian +lwo I L Luwo +lwt I L Lewotobi +lwu I L Lawu +lww I L Lewo +lya I L Layakha +lyg I L Lyngngam +lyn I L Luyana +lzh I H Literary Chinese +lzl I L Litzlitz +lzn I L Leinong Naga +lzz I L Laz +maa I L San Jerónimo Tecóatl Mazatec +mab I L Yutanduchi Mixtec +mad mad mad I L Madurese +mae I L Bo-Rukul +maf I L Mafa +mag mag mag I L Magahi +mah mah mah mh I L Marshallese +mai mai mai I L Maithili +maj I L Jalapa De Díaz Mazatec +mak mak mak I L Makasar +mal mal mal ml I L Malayalam +mam I L Mam +man man man M L Mandingo +maq I L Chiquihuitlán Mazatec +mar mar mar mr I L Marathi +mas mas mas I L Masai +mat I L San Francisco Matlatzinca +mau I L Huautla Mazatec +mav I L Sateré-Mawé +maw I L Mampruli +max I L North Moluccan Malay +maz I L Central Mazahua +mba I L Higaonon +mbb I L Western Bukidnon Manobo +mbc I L Macushi +mbd I L Dibabawon Manobo +mbe I E Molale +mbf I L Baba Malay +mbh I L Mangseng +mbi I L Ilianen Manobo +mbj I L Nadëb +mbk I L Malol +mbl I L Maxakalí +mbm I L Ombamba +mbn I L Macaguán +mbo I L Mbo (Cameroon) +mbp I L Malayo +mbq I L Maisin +mbr I L Nukak Makú +mbs I L Sarangani Manobo +mbt I L Matigsalug Manobo +mbu I L Mbula-Bwazza +mbv I L Mbulungish +mbw I L Maring +mbx I L Mari (East Sepik Province) +mby I L Memoni +mbz I L Amoltepec Mixtec +mca I L Maca +mcb I L Machiguenga +mcc I L Bitur +mcd I L Sharanahua +mce I L Itundujia Mixtec +mcf I L Matsés +mcg I L Mapoyo +mch I L Maquiritari +mci I L Mese +mcj I L Mvanip +mck I L Mbunda +mcl I E Macaguaje +mcm I L Malaccan Creole Portuguese +mcn I L Masana +mco I L Coatlán Mixe +mcp I L Makaa +mcq I L Ese +mcr I L Menya +mcs I L Mambai +mct I L Mengisa +mcu I L Cameroon Mambila +mcv I L Minanibai +mcw I L Mawa (Chad) +mcx I L Mpiemo +mcy I L South Watut +mcz I L Mawan +mda I L Mada (Nigeria) +mdb I L Morigi +mdc I L Male (Papua New Guinea) +mdd I L Mbum +mde I L Maba (Chad) +mdf mdf mdf I L Moksha +mdg I L Massalat +mdh I L Maguindanaon +mdi I L Mamvu +mdj I L Mangbetu +mdk I L Mangbutu +mdl I L Maltese Sign Language +mdm I L Mayogo +mdn I L Mbati +mdp I L Mbala +mdq I L Mbole +mdr mdr mdr I L Mandar +mds I L Maria (Papua New Guinea) +mdt I L Mbere +mdu I L Mboko +mdv I L Santa Lucía Monteverde Mixtec +mdw I L Mbosi +mdx I L Dizin +mdy I L Male (Ethiopia) +mdz I L Suruí Do Pará +mea I L Menka +meb I L Ikobi +mec I L Mara +med I L Melpa +mee I L Mengen +mef I L Megam +meh I L Southwestern Tlaxiaco Mixtec +mei I L Midob +mej I L Meyah +mek I L Mekeo +mel I L Central Melanau +mem I E Mangala +men men men I L Mende (Sierra Leone) +meo I L Kedah Malay +mep I L Miriwung +meq I L Merey +mer I L Meru +mes I L Masmaje +met I L Mato +meu I L Motu +mev I L Mano +mew I L Maaka +mey I L Hassaniyya +mez I L Menominee +mfa I L Pattani Malay +mfb I L Bangka +mfc I L Mba +mfd I L Mendankwe-Nkwen +mfe I L Morisyen +mff I L Naki +mfg I L Mogofin +mfh I L Matal +mfi I L Wandala +mfj I L Mefele +mfk I L North Mofu +mfl I L Putai +mfm I L Marghi South +mfn I L Cross River Mbembe +mfo I L Mbe +mfp I L Makassar Malay +mfq I L Moba +mfr I L Marithiel +mfs I L Mexican Sign Language +mft I L Mokerang +mfu I L Mbwela +mfv I L Mandjak +mfw I E Mulaha +mfx I L Melo +mfy I L Mayo +mfz I L Mabaan +mga mga mga I H Middle Irish (900-1200) +mgb I L Mararit +mgc I L Morokodo +mgd I L Moru +mge I L Mango +mgf I L Maklew +mgg I L Mpumpong +mgh I L Makhuwa-Meetto +mgi I L Lijili +mgj I L Abureni +mgk I L Mawes +mgl I L Maleu-Kilenge +mgm I L Mambae +mgn I L Mbangi +mgo I L Meta' +mgp I L Eastern Magar +mgq I L Malila +mgr I L Mambwe-Lungu +mgs I L Manda (Tanzania) +mgt I L Mongol +mgu I L Mailu +mgv I L Matengo +mgw I L Matumbi +mgy I L Mbunga +mgz I L Mbugwe +mha I L Manda (India) +mhb I L Mahongwe +mhc I L Mocho +mhd I L Mbugu +mhe I L Besisi +mhf I L Mamaa +mhg I L Margu +mhh I L Maskoy Pidgin +mhi I L Ma'di +mhj I L Mogholi +mhk I L Mungaka +mhl I L Mauwake +mhm I L Makhuwa-Moniga +mhn I L Mócheno +mho I L Mashi (Zambia) +mhp I L Balinese Malay +mhq I L Mandan +mhr I L Eastern Mari +mhs I L Buru (Indonesia) +mht I L Mandahuaca +mhu I L Digaro-Mishmi +mhw I L Mbukushu +mhx I L Maru +mhy I L Ma'anyan +mhz I L Mor (Mor Islands) +mia I L Miami +mib I L Atatláhuca Mixtec +mic mic mic I L Mi'kmaq +mid I L Mandaic +mie I L Ocotepec Mixtec +mif I L Mofu-Gudur +mig I L San Miguel El Grande Mixtec +mih I L Chayuco Mixtec +mii I L Chigmecatitlán Mixtec +mij I L Abar +mik I L Mikasuki +mil I L Peñoles Mixtec +mim I L Alacatlatzala Mixtec +min min min I L Minangkabau +mio I L Pinotepa Nacional Mixtec +mip I L Apasco-Apoala Mixtec +miq I L Mískito +mir I L Isthmus Mixe +mis mis mis S S Uncoded languages +mit I L Southern Puebla Mixtec +miu I L Cacaloxtepec Mixtec +miw I L Akoye +mix I L Mixtepec Mixtec +miy I L Ayutla Mixtec +miz I L Coatzospan Mixtec +mjc I L San Juan Colorado Mixtec +mjd I L Northwest Maidu +mje I E Muskum +mjg I L Tu +mjh I L Mwera (Nyasa) +mji I L Kim Mun +mjj I L Mawak +mjk I L Matukar +mjl I L Mandeali +mjm I L Medebur +mjn I L Ma (Papua New Guinea) +mjo I L Malankuravan +mjp I L Malapandaram +mjq I E Malaryan +mjr I L Malavedan +mjs I L Miship +mjt I L Sauria Paharia +mju I L Manna-Dora +mjv I L Mannan +mjw I L Karbi +mjx I L Mahali +mjy I E Mahican +mjz I L Majhi +mka I L Mbre +mkb I L Mal Paharia +mkc I L Siliput +mkd mac mkd mk I L Macedonian +mke I L Mawchi +mkf I L Miya +mkg I L Mak (China) +mki I L Dhatki +mkj I L Mokilese +mkk I L Byep +mkl I L Mokole +mkm I L Moklen +mkn I L Kupang Malay +mko I L Mingang Doso +mkp I L Moikodi +mkq I E Bay Miwok +mkr I L Malas +mks I L Silacayoapan Mixtec +mkt I L Vamale +mku I L Konyanka Maninka +mkv I L Mafea +mkw I L Kituba (Congo) +mkx I L Kinamiging Manobo +mky I L East Makian +mkz I L Makasae +mla I L Malo +mlb I L Mbule +mlc I L Cao Lan +mle I L Manambu +mlf I L Mal +mlg mlg mlg mg M L Malagasy +mlh I L Mape +mli I L Malimpung +mlj I L Miltu +mlk I L Ilwana +mll I L Malua Bay +mlm I L Mulam +mln I L Malango +mlo I L Mlomp +mlp I L Bargam +mlq I L Western Maninkakan +mlr I L Vame +mls I L Masalit +mlt mlt mlt mt I L Maltese +mlu I L To'abaita +mlv I L Motlav +mlw I L Moloko +mlx I L Malfaxal +mlz I L Malaynon +mma I L Mama +mmb I L Momina +mmc I L Michoacán Mazahua +mmd I L Maonan +mme I L Mae +mmf I L Mundat +mmg I L North Ambrym +mmh I L Mehináku +mmi I L Musar +mmj I L Majhwar +mmk I L Mukha-Dora +mml I L Man Met +mmm I L Maii +mmn I L Mamanwa +mmo I L Mangga Buang +mmp I L Siawi +mmq I L Musak +mmr I L Western Xiangxi Miao +mmt I L Malalamai +mmu I L Mmaala +mmv I E Miriti +mmw I L Emae +mmx I L Madak +mmy I L Migaama +mmz I L Mabaale +mna I L Mbula +mnb I L Muna +mnc mnc mnc I L Manchu +mnd I L Mondé +mne I L Naba +mnf I L Mundani +mng I L Eastern Mnong +mnh I L Mono (Democratic Republic of Congo) +mni mni mni I L Manipuri +mnj I L Munji +mnk I L Mandinka +mnl I L Tiale +mnm I L Mapena +mnn I L Southern Mnong +mnp I L Min Bei Chinese +mnq I L Minriq +mnr I L Mono (USA) +mns I L Mansi +mnu I L Mer +mnv I L Rennell-Bellona +mnw I L Mon +mnx I L Manikion +mny I L Manyawa +mnz I L Moni +moa I L Mwan +moc I L Mocoví +mod I E Mobilian +moe I L Montagnais +mog I L Mongondow +moh moh moh I L Mohawk +moi I L Mboi +moj I L Monzombo +mok I L Morori +mom I E Mangue +mon mon mon mn M L Mongolian +moo I L Monom +mop I L Mopán Maya +moq I L Mor (Bomberai Peninsula) +mor I L Moro +mos mos mos I L Mossi +mot I L Barí +mou I L Mogum +mov I L Mohave +mow I L Moi (Congo) +mox I L Molima +moy I L Shekkacho +moz I L Mukulu +mpa I L Mpoto +mpb I L Mullukmulluk +mpc I L Mangarayi +mpd I L Machinere +mpe I L Majang +mpg I L Marba +mph I L Maung +mpi I L Mpade +mpj I L Martu Wangka +mpk I L Mbara (Chad) +mpl I L Middle Watut +mpm I L Yosondúa Mixtec +mpn I L Mindiri +mpo I L Miu +mpp I L Migabac +mpq I L Matís +mpr I L Vangunu +mps I L Dadibi +mpt I L Mian +mpu I L Makuráp +mpv I L Mungkip +mpw I L Mapidian +mpx I L Misima-Panaeati +mpy I L Mapia +mpz I L Mpi +mqa I L Maba (Indonesia) +mqb I L Mbuko +mqc I L Mangole +mqe I L Matepi +mqf I L Momuna +mqg I L Kota Bangun Kutai Malay +mqh I L Tlazoyaltepec Mixtec +mqi I L Mariri +mqj I L Mamasa +mqk I L Rajah Kabunsuwan Manobo +mql I L Mbelime +mqm I L South Marquesan +mqn I L Moronene +mqo I L Modole +mqp I L Manipa +mqq I L Minokok +mqr I L Mander +mqs I L West Makian +mqt I L Mok +mqu I L Mandari +mqv I L Mosimo +mqw I L Murupi +mqx I L Mamuju +mqy I L Manggarai +mqz I L Pano +mra I L Mlabri +mrb I L Marino +mrc I L Maricopa +mrd I L Western Magar +mre I E Martha's Vineyard Sign Language +mrf I L Elseng +mrg I L Mising +mrh I L Mara Chin +mri mao mri mi I L Maori +mrj I L Western Mari +mrk I L Hmwaveke +mrl I L Mortlockese +mrm I L Merlav +mrn I L Cheke Holo +mro I L Mru +mrp I L Morouas +mrq I L North Marquesan +mrr I L Maria (India) +mrs I L Maragus +mrt I L Marghi Central +mru I L Mono (Cameroon) +mrv I L Mangareva +mrw I L Maranao +mrx I L Maremgi +mry I L Mandaya +mrz I L Marind +msa may msa ms M L Malay (macrolanguage) +msb I L Masbatenyo +msc I L Sankaran Maninka +msd I L Yucatec Maya Sign Language +mse I L Musey +msf I L Mekwei +msg I L Moraid +msh I L Masikoro Malagasy +msi I L Sabah Malay +msj I L Ma (Democratic Republic of Congo) +msk I L Mansaka +msl I L Molof +msm I L Agusan Manobo +msn I L Vurës +mso I L Mombum +msp I E Maritsauá +msq I L Caac +msr I L Mongolian Sign Language +mss I L West Masela +msu I L Musom +msv I L Maslam +msw I L Mansoanka +msx I L Moresada +msy I L Aruamu +msz I L Momare +mta I L Cotabato Manobo +mtb I L Anyin Morofo +mtc I L Munit +mtd I L Mualang +mte I L Mono (Solomon Islands) +mtf I L Murik (Papua New Guinea) +mtg I L Una +mth I L Munggui +mti I L Maiwa (Papua New Guinea) +mtj I L Moskona +mtk I L Mbe' +mtl I L Montol +mtm I E Mator +mtn I E Matagalpa +mto I L Totontepec Mixe +mtp I L Wichí Lhamtés Nocten +mtq I L Muong +mtr I L Mewari +mts I L Yora +mtt I L Mota +mtu I L Tututepec Mixtec +mtv I L Asaro'o +mtw I L Southern Binukidnon +mtx I L Tidaá Mixtec +mty I L Nabi +mua I L Mundang +mub I L Mubi +muc I L Ajumbu +mud I L Mednyj Aleut +mue I L Media Lengua +mug I L Musgu +muh I L Mündü +mui I L Musi +muj I L Mabire +muk I L Mugom +mul mul mul S S Multiple languages +mum I L Maiwala +muo I L Nyong +mup I L Malvi +muq I L Eastern Xiangxi Miao +mur I L Murle +mus mus mus I L Creek +mut I L Western Muria +muu I L Yaaku +muv I L Muthuvan +mux I L Bo-Ung +muy I L Muyang +muz I L Mursi +mva I L Manam +mvb I E Mattole +mvd I L Mamboru +mve I L Marwari (Pakistan) +mvf I L Peripheral Mongolian +mvg I L Yucuañe Mixtec +mvh I L Mulgi +mvi I L Miyako +mvk I L Mekmek +mvl I E Mbara (Australia) +mvm I L Muya +mvn I L Minaveha +mvo I L Marovo +mvp I L Duri +mvq I L Moere +mvr I L Marau +mvs I L Massep +mvt I L Mpotovoro +mvu I L Marfa +mvv I L Tagal Murut +mvw I L Machinga +mvx I L Meoswar +mvy I L Indus Kohistani +mvz I L Mesqan +mwa I L Mwatebu +mwb I L Juwal +mwc I L Are +mwe I L Mwera (Chimwera) +mwf I L Murrinh-Patha +mwg I L Aiklep +mwh I L Mouk-Aria +mwi I L Labo +mwj I L Maligo +mwk I L Kita Maninkakan +mwl mwl mwl I L Mirandese +mwm I L Sar +mwn I L Nyamwanga +mwo I L Central Maewo +mwp I L Kala Lagaw Ya +mwq I L Mün Chin +mwr mwr mwr M L Marwari +mws I L Mwimbi-Muthambi +mwt I L Moken +mwu I E Mittu +mwv I L Mentawai +mww I L Hmong Daw +mwx I L Mediak +mwy I L Mosiro +mwz I L Moingi +mxa I L Northwest Oaxaca Mixtec +mxb I L Tezoatlán Mixtec +mxc I L Manyika +mxd I L Modang +mxe I L Mele-Fila +mxf I L Malgbe +mxg I L Mbangala +mxh I L Mvuba +mxi I E Mozarabic +mxj I L Miju-Mishmi +mxk I L Monumbo +mxl I L Maxi Gbe +mxm I L Meramera +mxn I L Moi (Indonesia) +mxo I L Mbowe +mxp I L Tlahuitoltepec Mixe +mxq I L Juquila Mixe +mxr I L Murik (Malaysia) +mxs I L Huitepec Mixtec +mxt I L Jamiltepec Mixtec +mxu I L Mada (Cameroon) +mxv I L Metlatónoc Mixtec +mxw I L Namo +mxx I L Mahou +mxy I L Southeastern Nochixtlán Mixtec +mxz I L Central Masela +mya bur mya my I L Burmese +myb I L Mbay +myc I L Mayeka +myd I L Maramba +mye I L Myene +myf I L Bambassi +myg I L Manta +myh I L Makah +myi I L Mina (India) +myj I L Mangayat +myk I L Mamara Senoufo +myl I L Moma +mym I L Me'en +myo I L Anfillo +myp I L Pirahã +myr I L Muniche +mys I E Mesmes +myu I L Mundurukú +myv myv myv I L Erzya +myw I L Muyuw +myx I L Masaaba +myy I L Macuna +myz I H Classical Mandaic +mza I L Santa María Zacatepec Mixtec +mzb I L Tumzabt +mzc I L Madagascar Sign Language +mzd I L Malimba +mze I L Morawa +mzg I L Monastic Sign Language +mzh I L Wichí Lhamtés Güisnay +mzi I L Ixcatlán Mazatec +mzj I L Manya +mzk I L Nigeria Mambila +mzl I L Mazatlán Mixe +mzm I L Mumuye +mzn I L Mazanderani +mzo I E Matipuhy +mzp I L Movima +mzq I L Mori Atas +mzr I L Marúbo +mzs I L Macanese +mzt I L Mintil +mzu I L Inapang +mzv I L Manza +mzw I L Deg +mzx I L Mawayana +mzy I L Mozambican Sign Language +mzz I L Maiadomu +naa I L Namla +nab I L Southern Nambikuára +nac I L Narak +nad I L Nijadali +nae I L Naka'ela +naf I L Nabak +nag I L Naga Pidgin +naj I L Nalu +nak I L Nakanai +nal I L Nalik +nam I L Ngan'gityemerri +nan I L Min Nan Chinese +nao I L Naaba +nap nap nap I L Neapolitan +naq I L Nama (Namibia) +nar I L Iguta +nas I L Naasioi +nat I L Hungworo +nau nau nau na I L Nauru +nav nav nav nv I L Navajo +naw I L Nawuri +nax I L Nakwi +nay I E Narrinyeri +naz I L Coatepec Nahuatl +nba I L Nyemba +nbb I L Ndoe +nbc I L Chang Naga +nbd I L Ngbinda +nbe I L Konyak Naga +nbg I L Nagarchal +nbh I L Ngamo +nbi I L Mao Naga +nbj I L Ngarinman +nbk I L Nake +nbl nbl nbl nr I L South Ndebele +nbm I L Ngbaka Ma'bo +nbn I L Kuri +nbo I L Nkukoli +nbp I L Nnam +nbq I L Nggem +nbr I L Numana-Nunku-Gbantu-Numbu +nbs I L Namibian Sign Language +nbt I L Na +nbu I L Rongmei Naga +nbv I L Ngamambo +nbw I L Southern Ngbandi +nby I L Ningera +nca I L Iyo +ncb I L Central Nicobarese +ncc I L Ponam +ncd I L Nachering +nce I L Yale +ncf I L Notsi +ncg I L Nisga'a +nch I L Central Huasteca Nahuatl +nci I H Classical Nahuatl +ncj I L Northern Puebla Nahuatl +nck I L Nakara +ncl I L Michoacán Nahuatl +ncm I L Nambo +ncn I L Nauna +nco I L Sibe +ncp I L Ndaktup +ncr I L Ncane +ncs I L Nicaraguan Sign Language +nct I L Chothe Naga +ncu I L Chumburung +ncx I L Central Puebla Nahuatl +ncz I E Natchez +nda I L Ndasa +ndb I L Kenswei Nsei +ndc I L Ndau +ndd I L Nde-Nsele-Nta +nde nde nde nd I L North Ndebele +ndf I E Nadruvian +ndg I L Ndengereko +ndh I L Ndali +ndi I L Samba Leko +ndj I L Ndamba +ndk I L Ndaka +ndl I L Ndolo +ndm I L Ndam +ndn I L Ngundi +ndo ndo ndo ng I L Ndonga +ndp I L Ndo +ndq I L Ndombe +ndr I L Ndoola +nds nds nds I L Low German +ndt I L Ndunga +ndu I L Dugun +ndv I L Ndut +ndw I L Ndobo +ndx I L Nduga +ndy I L Lutos +ndz I L Ndogo +nea I L Eastern Ngad'a +neb I L Toura (Côte d'Ivoire) +nec I L Nedebang +ned I L Nde-Gbite +nee I L Nêlêmwa-Nixumwak +nef I L Nefamese +neg I L Negidal +neh I L Nyenkha +nei I A Neo-Hittite +nej I L Neko +nek I L Neku +nem I L Nemi +nen I L Nengone +neo I L Ná-Meo +nep nep nep ne M L Nepali (macrolanguage) +neq I L North Central Mixe +ner I L Yahadian +nes I L Bhoti Kinnauri +net I L Nete +neu I C Neo +nev I L Nyaheun +new new new I L Newari +nex I L Neme +ney I L Neyo +nez I L Nez Perce +nfa I L Dhao +nfd I L Ahwai +nfl I L Ayiwo +nfr I L Nafaanra +nfu I L Mfumte +nga I L Ngbaka +ngb I L Northern Ngbandi +ngc I L Ngombe (Democratic Republic of Congo) +ngd I L Ngando (Central African Republic) +nge I L Ngemba +ngg I L Ngbaka Manza +ngh I L N/u +ngi I L Ngizim +ngj I L Ngie +ngk I L Dalabon +ngl I L Lomwe +ngm I L Ngatik Men's Creole +ngn I L Ngwo +ngo I L Ngoni +ngp I L Ngulu +ngq I L Ngurimi +ngr I L Engdewu +ngs I L Gvoko +ngt I L Ngeq +ngu I L Guerrero Nahuatl +ngv I E Nagumi +ngw I L Ngwaba +ngx I L Nggwahyi +ngy I L Tibea +ngz I L Ngungwel +nha I L Nhanda +nhb I L Beng +nhc I E Tabasco Nahuatl +nhd I L Chiripá +nhe I L Eastern Huasteca Nahuatl +nhf I L Nhuwala +nhg I L Tetelcingo Nahuatl +nhh I L Nahari +nhi I L Zacatlán-Ahuacatlán-Tepetzintla Nahuatl +nhk I L Isthmus-Cosoleacaque Nahuatl +nhm I L Morelos Nahuatl +nhn I L Central Nahuatl +nho I L Takuu +nhp I L Isthmus-Pajapan Nahuatl +nhq I L Huaxcaleca Nahuatl +nhr I L Naro +nht I L Ometepec Nahuatl +nhu I L Noone +nhv I L Temascaltepec Nahuatl +nhw I L Western Huasteca Nahuatl +nhx I L Isthmus-Mecayapan Nahuatl +nhy I L Northern Oaxaca Nahuatl +nhz I L Santa María La Alta Nahuatl +nia nia nia I L Nias +nib I L Nakame +nid I E Ngandi +nie I L Niellim +nif I L Nek +nig I E Ngalakan +nih I L Nyiha (Tanzania) +nii I L Nii +nij I L Ngaju +nik I L Southern Nicobarese +nil I L Nila +nim I L Nilamba +nin I L Ninzo +nio I L Nganasan +niq I L Nandi +nir I L Nimboran +nis I L Nimi +nit I L Southeastern Kolami +niu niu niu I L Niuean +niv I L Gilyak +niw I L Nimo +nix I L Hema +niy I L Ngiti +niz I L Ningil +nja I L Nzanyi +njb I L Nocte Naga +njd I L Ndonde Hamba +njh I L Lotha Naga +nji I L Gudanji +njj I L Njen +njl I L Njalgulgule +njm I L Angami Naga +njn I L Liangmai Naga +njo I L Ao Naga +njr I L Njerep +njs I L Nisa +njt I L Ndyuka-Trio Pidgin +nju I L Ngadjunmaya +njx I L Kunyi +njy I L Njyem +njz I L Nyishi +nka I L Nkoya +nkb I L Khoibu Naga +nkc I L Nkongho +nkd I L Koireng +nke I L Duke +nkf I L Inpui Naga +nkg I L Nekgini +nkh I L Khezha Naga +nki I L Thangal Naga +nkj I L Nakai +nkk I L Nokuku +nkm I L Namat +nkn I L Nkangala +nko I L Nkonya +nkp I E Niuatoputapu +nkq I L Nkami +nkr I L Nukuoro +nks I L North Asmat +nkt I L Nyika (Tanzania) +nku I L Bouna Kulango +nkv I L Nyika (Malawi and Zambia) +nkw I L Nkutu +nkx I L Nkoroo +nkz I L Nkari +nla I L Ngombale +nlc I L Nalca +nld dut nld nl I L Dutch +nle I L East Nyala +nlg I L Gela +nli I L Grangali +nlj I L Nyali +nlk I L Ninia Yali +nll I L Nihali +nlo I L Ngul +nlq I L Lao Naga +nlu I L Nchumbulu +nlv I L Orizaba Nahuatl +nlw I E Walangama +nlx I L Nahali +nly I L Nyamal +nlz I L Nalögo +nma I L Maram Naga +nmb I L Big Nambas +nmc I L Ngam +nmd I L Ndumu +nme I L Mzieme Naga +nmf I L Tangkhul Naga (India) +nmg I L Kwasio +nmh I L Monsang Naga +nmi I L Nyam +nmj I L Ngombe (Central African Republic) +nmk I L Namakura +nml I L Ndemli +nmm I L Manangba +nmn I L !Xóõ +nmo I L Moyon Naga +nmp I E Nimanbur +nmq I L Nambya +nmr I E Nimbari +nms I L Letemboi +nmt I L Namonuito +nmu I L Northeast Maidu +nmv I E Ngamini +nmw I L Nimoa +nmx I L Nama (Papua New Guinea) +nmy I L Namuyi +nmz I L Nawdm +nna I L Nyangumarta +nnb I L Nande +nnc I L Nancere +nnd I L West Ambae +nne I L Ngandyera +nnf I L Ngaing +nng I L Maring Naga +nnh I L Ngiemboon +nni I L North Nuaulu +nnj I L Nyangatom +nnk I L Nankina +nnl I L Northern Rengma Naga +nnm I L Namia +nnn I L Ngete +nno nno nno nn I L Norwegian Nynorsk +nnp I L Wancho Naga +nnq I L Ngindo +nnr I E Narungga +nns I L Ningye +nnt I E Nanticoke +nnu I L Dwang +nnv I E Nugunu (Australia) +nnw I L Southern Nuni +nnx I L Ngong +nny I E Nyangga +nnz I L Nda'nda' +noa I L Woun Meu +nob nob nob nb I L Norwegian Bokmål +noc I L Nuk +nod I L Northern Thai +noe I L Nimadi +nof I L Nomane +nog nog nog I L Nogai +noh I L Nomu +noi I L Noiri +noj I L Nonuya +nok I E Nooksack +nol I E Nomlaki +nom I E Nocamán +non non non I H Old Norse +nop I L Numanggang +noq I L Ngongo +nor nor nor no M L Norwegian +nos I L Eastern Nisu +not I L Nomatsiguenga +nou I L Ewage-Notu +nov I C Novial +now I L Nyambo +noy I L Noy +noz I L Nayi +npa I L Nar Phu +npb I L Nupbikha +npg I L Ponyo-Gongwang Naga +nph I L Phom Naga +npi I L Nepali (individual language) +npl I L Southeastern Puebla Nahuatl +npn I L Mondropolon +npo I L Pochuri Naga +nps I L Nipsan +npu I L Puimei Naga +npy I L Napu +nqg I L Southern Nago +nqk I L Kura Ede Nago +nqm I L Ndom +nqn I L Nen +nqo nqo nqo I L N'Ko +nqq I L Kyan-Karyaw Naga +nqy I L Akyaung Ari Naga +nra I L Ngom +nrb I L Nara +nrc I A Noric +nre I L Southern Rengma Naga +nrg I L Narango +nri I L Chokri Naga +nrk I L Ngarla +nrl I L Ngarluma +nrm I L Narom +nrn I E Norn +nrp I A North Picene +nrr I E Norra +nrt I E Northern Kalapuya +nru I L Narua +nrx I E Ngurmbur +nrz I L Lala +nsa I L Sangtam Naga +nsc I L Nshi +nsd I L Southern Nisu +nse I L Nsenga +nsf I L Northwestern Nisu +nsg I L Ngasa +nsh I L Ngoshie +nsi I L Nigerian Sign Language +nsk I L Naskapi +nsl I L Norwegian Sign Language +nsm I L Sumi Naga +nsn I L Nehan +nso nso nso I L Pedi +nsp I L Nepalese Sign Language +nsq I L Northern Sierra Miwok +nsr I L Maritime Sign Language +nss I L Nali +nst I L Tase Naga +nsu I L Sierra Negra Nahuatl +nsv I L Southwestern Nisu +nsw I L Navut +nsx I L Nsongo +nsy I L Nasal +nsz I L Nisenan +nte I L Nathembo +ntg I E Ngantangarra +nti I L Natioro +ntj I L Ngaanyatjarra +ntk I L Ikoma-Nata-Isenye +ntm I L Nateni +nto I L Ntomba +ntp I L Northern Tepehuan +ntr I L Delo +nts I E Natagaimas +ntu I L Natügu +ntw I E Nottoway +ntx I L Tangkhul Naga (Myanmar) +nty I L Mantsi +ntz I L Natanzi +nua I L Yuanga +nuc I E Nukuini +nud I L Ngala +nue I L Ngundu +nuf I L Nusu +nug I E Nungali +nuh I L Ndunda +nui I L Ngumbi +nuj I L Nyole +nuk I L Nuu-chah-nulth +nul I L Nusa Laut +num I L Niuafo'ou +nun I L Anong +nuo I L Nguôn +nup I L Nupe-Nupe-Tako +nuq I L Nukumanu +nur I L Nukuria +nus I L Nuer +nut I L Nung (Viet Nam) +nuu I L Ngbundu +nuv I L Northern Nuni +nuw I L Nguluwan +nux I L Mehek +nuy I L Nunggubuyu +nuz I L Tlamacazapa Nahuatl +nvh I L Nasarian +nvm I L Namiae +nvo I L Nyokon +nwa I E Nawathinehena +nwb I L Nyabwa +nwc nwc nwc I H Classical Newari +nwe I L Ngwe +nwg I E Ngayawung +nwi I L Southwest Tanna +nwm I L Nyamusa-Molo +nwo I E Nauo +nwr I L Nawaru +nwx I H Middle Newar +nwy I E Nottoway-Meherrin +nxa I L Nauete +nxd I L Ngando (Democratic Republic of Congo) +nxe I L Nage +nxg I L Ngad'a +nxi I L Nindi +nxk I L Koki Naga +nxl I L South Nuaulu +nxm I A Numidian +nxn I E Ngawun +nxq I L Naxi +nxr I L Ninggerum +nxu I E Narau +nxx I L Nafri +nya nya nya ny I L Nyanja +nyb I L Nyangbo +nyc I L Nyanga-li +nyd I L Nyore +nye I L Nyengo +nyf I L Giryama +nyg I L Nyindu +nyh I L Nyigina +nyi I L Ama (Sudan) +nyj I L Nyanga +nyk I L Nyaneka +nyl I L Nyeu +nym nym nym I L Nyamwezi +nyn nyn nyn I L Nyankole +nyo nyo nyo I L Nyoro +nyp I E Nyang'i +nyq I L Nayini +nyr I L Nyiha (Malawi) +nys I L Nyunga +nyt I E Nyawaygi +nyu I L Nyungwe +nyv I E Nyulnyul +nyw I L Nyaw +nyx I E Nganyaywana +nyy I L Nyakyusa-Ngonde +nza I L Tigon Mbembe +nzb I L Njebi +nzi nzi nzi I L Nzima +nzk I L Nzakara +nzm I L Zeme Naga +nzs I L New Zealand Sign Language +nzu I L Teke-Nzikou +nzy I L Nzakambay +nzz I L Nanga Dama Dogon +oaa I L Orok +oac I L Oroch +oar I A Old Aramaic (up to 700 BCE) +oav I H Old Avar +obi I E Obispeño +obk I L Southern Bontok +obl I L Oblo +obm I A Moabite +obo I L Obo Manobo +obr I H Old Burmese +obt I H Old Breton +obu I L Obulom +oca I L Ocaina +och I A Old Chinese +oci oci oci oc I L Occitan (post 1500) +oco I H Old Cornish +ocu I L Atzingo Matlatzinca +oda I L Odut +odk I L Od +odt I H Old Dutch +odu I L Odual +ofo I E Ofo +ofs I H Old Frisian +ofu I L Efutop +ogb I L Ogbia +ogc I L Ogbah +oge I H Old Georgian +ogg I L Ogbogolo +ogo I L Khana +ogu I L Ogbronuagum +oht I A Old Hittite +ohu I H Old Hungarian +oia I L Oirata +oin I L Inebu One +ojb I L Northwestern Ojibwa +ojc I L Central Ojibwa +ojg I L Eastern Ojibwa +oji oji oji oj M L Ojibwa +ojp I H Old Japanese +ojs I L Severn Ojibwa +ojv I L Ontong Java +ojw I L Western Ojibwa +oka I L Okanagan +okb I L Okobo +okd I L Okodia +oke I L Okpe (Southwestern Edo) +okg I E Koko Babangk +okh I L Koresh-e Rostam +oki I L Okiek +okj I E Oko-Juwoi +okk I L Kwamtim One +okl I E Old Kentish Sign Language +okm I H Middle Korean (10th-16th cent.) +okn I L Oki-No-Erabu +oko I H Old Korean (3rd-9th cent.) +okr I L Kirike +oks I L Oko-Eni-Osayen +oku I L Oku +okv I L Orokaiva +okx I L Okpe (Northwestern Edo) +ola I L Walungge +old I L Mochi +ole I L Olekha +olk I E Olkol +olm I L Oloma +olo I L Livvi +olr I L Olrat +oma I L Omaha-Ponca +omb I L East Ambae +omc I E Mochica +ome I E Omejes +omg I L Omagua +omi I L Omi +omk I E Omok +oml I L Ombo +omn I A Minoan +omo I L Utarmbung +omp I H Old Manipuri +omr I H Old Marathi +omt I L Omotik +omu I E Omurano +omw I L South Tairora +omx I H Old Mon +ona I L Ona +onb I L Lingao +one I L Oneida +ong I L Olo +oni I L Onin +onj I L Onjob +onk I L Kabore One +onn I L Onobasulu +ono I L Onondaga +onp I L Sartang +onr I L Northern One +ons I L Ono +ont I L Ontenu +onu I L Unua +onw I H Old Nubian +onx I L Onin Based Pidgin +ood I L Tohono O'odham +oog I L Ong +oon I L Önge +oor I L Oorlams +oos I A Old Ossetic +opa I L Okpamheri +opk I L Kopkaka +opm I L Oksapmin +opo I L Opao +opt I E Opata +opy I L Ofayé +ora I L Oroha +orc I L Orma +ore I L Orejón +org I L Oring +orh I L Oroqen +ori ori ori or M L Oriya (macrolanguage) +orm orm orm om M L Oromo +orn I L Orang Kanaq +oro I L Orokolo +orr I L Oruma +ors I L Orang Seletar +ort I L Adivasi Oriya +oru I L Ormuri +orv I H Old Russian +orw I L Oro Win +orx I L Oro +ory I L Oriya (individual language) +orz I L Ormu +osa osa osa I L Osage +osc I A Oscan +osi I L Osing +oso I L Ososo +osp I H Old Spanish +oss oss oss os I L Ossetian +ost I L Osatu +osu I L Southern One +osx I H Old Saxon +ota ota ota I H Ottoman Turkish (1500-1928) +otb I H Old Tibetan +otd I L Ot Danum +ote I L Mezquital Otomi +oti I E Oti +otk I H Old Turkish +otl I L Tilapa Otomi +otm I L Eastern Highland Otomi +otn I L Tenango Otomi +otq I L Querétaro Otomi +otr I L Otoro +ots I L Estado de México Otomi +ott I L Temoaya Otomi +otu I E Otuke +otw I L Ottawa +otx I L Texcatepec Otomi +oty I A Old Tamil +otz I L Ixtenco Otomi +oua I L Tagargrent +oub I L Glio-Oubi +oue I L Oune +oui I H Old Uighur +oum I E Ouma +oun I L !O!ung +owi I L Owiniga +owl I H Old Welsh +oyb I L Oy +oyd I L Oyda +oym I L Wayampi +oyy I L Oya'oya +ozm I L Koonzime +pab I L Parecís +pac I L Pacoh +pad I L Paumarí +pae I L Pagibete +paf I E Paranawát +pag pag pag I L Pangasinan +pah I L Tenharim +pai I L Pe +pak I L Parakanã +pal pal pal I A Pahlavi +pam pam pam I L Pampanga +pan pan pan pa I L Panjabi +pao I L Northern Paiute +pap pap pap I L Papiamento +paq I L Parya +par I L Panamint +pas I L Papasena +pat I L Papitalai +pau pau pau I L Palauan +pav I L Pakaásnovos +paw I L Pawnee +pax I E Pankararé +pay I L Pech +paz I E Pankararú +pbb I L Páez +pbc I L Patamona +pbe I L Mezontla Popoloca +pbf I L Coyotepec Popoloca +pbg I E Paraujano +pbh I L E'ñapa Woromaipu +pbi I L Parkwa +pbl I L Mak (Nigeria) +pbn I L Kpasam +pbo I L Papel +pbp I L Badyara +pbr I L Pangwa +pbs I L Central Pame +pbt I L Southern Pashto +pbu I L Northern Pashto +pbv I L Pnar +pby I L Pyu +pca I L Santa Inés Ahuatempan Popoloca +pcb I L Pear +pcc I L Bouyei +pcd I L Picard +pce I L Ruching Palaung +pcf I L Paliyan +pcg I L Paniya +pch I L Pardhan +pci I L Duruwa +pcj I L Parenga +pck I L Paite Chin +pcl I L Pardhi +pcm I L Nigerian Pidgin +pcn I L Piti +pcp I L Pacahuara +pcw I L Pyapun +pda I L Anam +pdc I L Pennsylvania German +pdi I L Pa Di +pdn I L Podena +pdo I L Padoe +pdt I L Plautdietsch +pdu I L Kayan +pea I L Peranakan Indonesian +peb I E Eastern Pomo +ped I L Mala (Papua New Guinea) +pee I L Taje +pef I E Northeastern Pomo +peg I L Pengo +peh I L Bonan +pei I L Chichimeca-Jonaz +pej I E Northern Pomo +pek I L Penchal +pel I L Pekal +pem I L Phende +peo peo peo I H Old Persian (ca. 600-400 B.C.) +pep I L Kunja +peq I L Southern Pomo +pes I L Iranian Persian +pev I L Pémono +pex I L Petats +pey I L Petjo +pez I L Eastern Penan +pfa I L Pááfang +pfe I L Peere +pfl I L Pfaelzisch +pga I L Sudanese Creole Arabic +pgg I L Pangwali +pgi I L Pagi +pgk I L Rerep +pgl I A Primitive Irish +pgn I A Paelignian +pgs I L Pangseng +pgu I L Pagu +pha I L Pa-Hng +phd I L Phudagi +phg I L Phuong +phh I L Phukha +phk I L Phake +phl I L Phalura +phm I L Phimbi +phn phn phn I A Phoenician +pho I L Phunoi +phq I L Phana' +phr I L Pahari-Potwari +pht I L Phu Thai +phu I L Phuan +phv I L Pahlavani +phw I L Phangduwali +pia I L Pima Bajo +pib I L Yine +pic I L Pinji +pid I L Piaroa +pie I E Piro +pif I L Pingelapese +pig I L Pisabo +pih I L Pitcairn-Norfolk +pii I L Pini +pij I E Pijao +pil I L Yom +pim I E Powhatan +pin I L Piame +pio I L Piapoco +pip I L Pero +pir I L Piratapuyo +pis I L Pijin +pit I E Pitta Pitta +piu I L Pintupi-Luritja +piv I L Pileni +piw I L Pimbwe +pix I L Piu +piy I L Piya-Kwonci +piz I L Pije +pjt I L Pitjantjatjara +pka I H Ardhamāgadhī Prākrit +pkb I L Pokomo +pkc I E Paekche +pkg I L Pak-Tong +pkh I L Pankhu +pkn I L Pakanha +pko I L Pökoot +pkp I L Pukapuka +pkr I L Attapady Kurumba +pks I L Pakistan Sign Language +pkt I L Maleng +pku I L Paku +pla I L Miani +plb I L Polonombauk +plc I L Central Palawano +pld I L Polari +ple I L Palu'e +plg I L Pilagá +plh I L Paulohi +pli pli pli pi I A Pali +plj I L Polci +plk I L Kohistani Shina +pll I L Shwe Palaung +pln I L Palenquero +plo I L Oluta Popoluca +plp I L Palpa +plq I A Palaic +plr I L Palaka Senoufo +pls I L San Marcos Tlalcoyalco Popoloca +plt I L Plateau Malagasy +plu I L Palikúr +plv I L Southwest Palawano +plw I L Brooke's Point Palawano +ply I L Bolyu +plz I L Paluan +pma I L Paama +pmb I L Pambia +pmc I E Palumata +pmd I E Pallanganmiddang +pme I L Pwaamei +pmf I L Pamona +pmh I H Māhārāṣṭri Prākrit +pmi I L Northern Pumi +pmj I L Southern Pumi +pmk I E Pamlico +pml I E Lingua Franca +pmm I L Pomo +pmn I L Pam +pmo I L Pom +pmq I L Northern Pame +pmr I L Paynamar +pms I L Piemontese +pmt I L Tuamotuan +pmu I L Mirpur Panjabi +pmw I L Plains Miwok +pmx I L Poumei Naga +pmy I L Papuan Malay +pmz I E Southern Pame +pna I L Punan Bah-Biau +pnb I L Western Panjabi +pnc I L Pannei +pne I L Western Penan +png I L Pongu +pnh I L Penrhyn +pni I L Aoheng +pnj I E Pinjarup +pnk I L Paunaka +pnl I L Paleni +pnm I L Punan Batu 1 +pnn I L Pinai-Hagahai +pno I E Panobo +pnp I L Pancana +pnq I L Pana (Burkina Faso) +pnr I L Panim +pns I L Ponosakan +pnt I L Pontic +pnu I L Jiongnai Bunu +pnv I L Pinigura +pnw I L Panytyima +pnx I L Phong-Kniang +pny I L Pinyin +pnz I L Pana (Central African Republic) +poc I L Poqomam +pod I E Ponares +poe I L San Juan Atzingo Popoloca +pof I L Poke +pog I E Potiguára +poh I L Poqomchi' +poi I L Highland Popoluca +pok I L Pokangá +pol pol pol pl I L Polish +pom I L Southeastern Pomo +pon pon pon I L Pohnpeian +poo I L Central Pomo +pop I L Pwapwâ +poq I L Texistepec Popoluca +por por por pt I L Portuguese +pos I L Sayula Popoluca +pot I L Potawatomi +pov I L Upper Guinea Crioulo +pow I L San Felipe Otlaltepec Popoloca +pox I E Polabian +poy I L Pogolo +ppa I L Pao +ppe I L Papi +ppi I L Paipai +ppk I L Uma +ppl I L Pipil +ppm I L Papuma +ppn I L Papapana +ppo I L Folopa +ppp I L Pelende +ppq I L Pei +pps I L San Luís Temalacayuca Popoloca +ppt I L Pare +ppu I E Papora +pqa I L Pa'a +pqm I L Malecite-Passamaquoddy +prb I L Lua' +prc I L Parachi +prd I L Parsi-Dari +pre I L Principense +prf I L Paranan +prg I L Prussian +prh I L Porohanon +pri I L Paicî +prk I L Parauk +prl I L Peruvian Sign Language +prm I L Kibiri +prn I L Prasuni +pro pro pro I H Old Provençal (to 1500) +prp I L Parsi +prq I L Ashéninka Perené +prr I E Puri +prs I L Dari +prt I L Phai +pru I L Puragi +prw I L Parawen +prx I L Purik +pry I L Pray 3 +prz I L Providencia Sign Language +psa I L Asue Awyu +psc I L Persian Sign Language +psd I L Plains Indian Sign Language +pse I L Central Malay +psg I L Penang Sign Language +psh I L Southwest Pashayi +psi I L Southeast Pashayi +psl I L Puerto Rican Sign Language +psm I E Pauserna +psn I L Panasuan +pso I L Polish Sign Language +psp I L Philippine Sign Language +psq I L Pasi +psr I L Portuguese Sign Language +pss I L Kaulong +pst I L Central Pashto +psu I H Sauraseni Prākrit +psw I L Port Sandwich +psy I E Piscataway +pta I L Pai Tavytera +pth I E Pataxó Hã-Ha-Hãe +pti I L Pintiini +ptn I L Patani +pto I L Zo'é +ptp I L Patep +ptr I L Piamatsina +ptt I L Enrekang +ptu I L Bambam +ptv I L Port Vato +ptw I E Pentlatch +pty I L Pathiya +pua I L Western Highland Purepecha +pub I L Purum +puc I L Punan Merap +pud I L Punan Aput +pue I L Puelche +puf I L Punan Merah +pug I L Phuie +pui I L Puinave +puj I L Punan Tubu +puk I L Pu Ko +pum I L Puma +puo I L Puoc +pup I L Pulabu +puq I E Puquina +pur I L Puruborá +pus pus pus ps M L Pushto +put I L Putoh +puu I L Punu +puw I L Puluwatese +pux I L Puare +puy I E Purisimeño +puz I L Purum Naga +pwa I L Pawaia +pwb I L Panawa +pwg I L Gapapaiwa +pwi I E Patwin +pwm I L Molbog +pwn I L Paiwan +pwo I L Pwo Western Karen +pwr I L Powari +pww I L Pwo Northern Karen +pxm I L Quetzaltepec Mixe +pye I L Pye Krumen +pym I L Fyam +pyn I L Poyanáwa +pys I L Paraguayan Sign Language +pyu I L Puyuma +pyx I A Pyu (Myanmar) +pyy I L Pyen +pzn I L Para Naga +qua I L Quapaw +qub I L Huallaga Huánuco Quechua +quc I L K'iche' +qud I L Calderón Highland Quichua +que que que qu M L Quechua +quf I L Lambayeque Quechua +qug I L Chimborazo Highland Quichua +quh I L South Bolivian Quechua +qui I L Quileute +quk I L Chachapoyas Quechua +qul I L North Bolivian Quechua +qum I L Sipacapense +qun I E Quinault +qup I L Southern Pastaza Quechua +quq I L Quinqui +qur I L Yanahuanca Pasco Quechua +qus I L Santiago del Estero Quichua +quv I L Sacapulteco +quw I L Tena Lowland Quichua +qux I L Yauyos Quechua +quy I L Ayacucho Quechua +quz I L Cusco Quechua +qva I L Ambo-Pasco Quechua +qvc I L Cajamarca Quechua +qve I L Eastern Apurímac Quechua +qvh I L Huamalíes-Dos de Mayo Huánuco Quechua +qvi I L Imbabura Highland Quichua +qvj I L Loja Highland Quichua +qvl I L Cajatambo North Lima Quechua +qvm I L Margos-Yarowilca-Lauricocha Quechua +qvn I L North Junín Quechua +qvo I L Napo Lowland Quechua +qvp I L Pacaraos Quechua +qvs I L San Martín Quechua +qvw I L Huaylla Wanca Quechua +qvy I L Queyu +qvz I L Northern Pastaza Quichua +qwa I L Corongo Ancash Quechua +qwc I H Classical Quechua +qwh I L Huaylas Ancash Quechua +qwm I E Kuman (Russia) +qws I L Sihuas Ancash Quechua +qwt I E Kwalhioqua-Tlatskanai +qxa I L Chiquián Ancash Quechua +qxc I L Chincha Quechua +qxh I L Panao Huánuco Quechua +qxl I L Salasaca Highland Quichua +qxn I L Northern Conchucos Ancash Quechua +qxo I L Southern Conchucos Ancash Quechua +qxp I L Puno Quechua +qxq I L Qashqa'i +qxr I L Cañar Highland Quichua +qxs I L Southern Qiang +qxt I L Santa Ana de Tusi Pasco Quechua +qxu I L Arequipa-La Unión Quechua +qxw I L Jauja Wanca Quechua +qya I C Quenya +qyp I E Quiripi +raa I L Dungmali +rab I L Camling +rac I L Rasawa +rad I L Rade +raf I L Western Meohang +rag I L Logooli +rah I L Rabha +rai I L Ramoaaina +raj raj raj M L Rajasthani +rak I L Tulu-Bohuai +ral I L Ralte +ram I L Canela +ran I L Riantana +rao I L Rao +rap rap rap I L Rapanui +raq I L Saam +rar rar rar I L Rarotongan +ras I L Tegali +rat I L Razajerdi +rau I L Raute +rav I L Sampang +raw I L Rawang +rax I L Rang +ray I L Rapa +raz I L Rahambuu +rbb I L Rumai Palaung +rbk I L Northern Bontok +rbl I L Miraya Bikol +rbp I E Barababaraba +rcf I L Réunion Creole French +rdb I L Rudbari +rea I L Rerau +reb I L Rembong +ree I L Rejang Kayan +reg I L Kara (Tanzania) +rei I L Reli +rej I L Rejang +rel I L Rendille +rem I E Remo +ren I L Rengao +rer I E Rer Bare +res I L Reshe +ret I L Retta +rey I L Reyesano +rga I L Roria +rge I L Romano-Greek +rgk I E Rangkas +rgn I L Romagnol +rgr I L Resígaro +rgs I L Southern Roglai +rgu I L Ringgou +rhg I L Rohingya +rhp I L Yahang +ria I L Riang (India) +rie I L Rien +rif I L Tarifit +ril I L Riang (Myanmar) +rim I L Nyaturu +rin I L Nungu +rir I L Ribun +rit I L Ritarungo +riu I L Riung +rjg I L Rajong +rji I L Raji +rjs I L Rajbanshi +rka I L Kraol +rkb I L Rikbaktsa +rkh I L Rakahanga-Manihiki +rki I L Rakhine +rkm I L Marka +rkt I L Rangpuri +rkw I E Arakwal +rma I L Rama +rmb I L Rembarunga +rmc I L Carpathian Romani +rmd I E Traveller Danish +rme I L Angloromani +rmf I L Kalo Finnish Romani +rmg I L Traveller Norwegian +rmh I L Murkim +rmi I L Lomavren +rmk I L Romkun +rml I L Baltic Romani +rmm I L Roma +rmn I L Balkan Romani +rmo I L Sinte Romani +rmp I L Rempi +rmq I L Caló +rms I L Romanian Sign Language +rmt I L Domari +rmu I L Tavringer Romani +rmv I C Romanova +rmw I L Welsh Romani +rmx I L Romam +rmy I L Vlax Romani +rmz I L Marma +rna I E Runa +rnd I L Ruund +rng I L Ronga +rnl I L Ranglong +rnn I L Roon +rnp I L Rongpo +rnr I E Nari Nari +rnw I L Rungwa +rob I L Tae' +roc I L Cacgia Roglai +rod I L Rogo +roe I L Ronji +rof I L Rombo +rog I L Northern Roglai +roh roh roh rm I L Romansh +rol I L Romblomanon +rom rom rom M L Romany +ron rum ron ro I L Romanian +roo I L Rotokas +rop I L Kriol +ror I L Rongga +rou I L Runga +row I L Dela-Oenale +rpn I L Repanbitip +rpt I L Rapting +rri I L Ririo +rro I L Waima +rrt I E Arritinngithigh +rsb I L Romano-Serbian +rsi I L Rennellese Sign Language +rsl I L Russian Sign Language +rtc I L Rungtu Chin +rth I L Ratahan +rtm I L Rotuman +rtw I L Rathawi +rub I L Gungu +ruc I L Ruuli +rue I L Rusyn +ruf I L Luguru +rug I L Roviana +ruh I L Ruga +rui I L Rufiji +ruk I L Che +run run run rn I L Rundi +ruo I L Istro Romanian +rup rup rup I L Macedo-Romanian +ruq I L Megleno Romanian +rus rus rus ru I L Russian +rut I L Rutul +ruu I L Lanas Lobu +ruy I L Mala (Nigeria) +ruz I L Ruma +rwa I L Rawo +rwk I L Rwa +rwm I L Amba (Uganda) +rwo I L Rawa +rwr I L Marwari (India) +rxd I L Ngardi +rxw I E Karuwali +ryn I L Northern Amami-Oshima +rys I L Yaeyama +ryu I L Central Okinawan +saa I L Saba +sab I L Buglere +sac I L Meskwaki +sad sad sad I L Sandawe +sae I L Sabanê +saf I L Safaliba +sag sag sag sg I L Sango +sah sah sah I L Yakut +saj I L Sahu +sak I L Sake +sam sam sam I E Samaritan Aramaic +san san san sa I A Sanskrit +sao I L Sause +sap I L Sanapaná +saq I L Samburu +sar I E Saraveca +sas sas sas I L Sasak +sat sat sat I L Santali +sau I L Saleman +sav I L Saafi-Saafi +saw I L Sawi +sax I L Sa +say I L Saya +saz I L Saurashtra +sba I L Ngambay +sbb I L Simbo +sbc I L Kele (Papua New Guinea) +sbd I L Southern Samo +sbe I L Saliba +sbf I L Shabo +sbg I L Seget +sbh I L Sori-Harengan +sbi I L Seti +sbj I L Surbakhal +sbk I L Safwa +sbl I L Botolan Sambal +sbm I L Sagala +sbn I L Sindhi Bhil +sbo I L Sabüm +sbp I L Sangu (Tanzania) +sbq I L Sileibi +sbr I L Sembakung Murut +sbs I L Subiya +sbt I L Kimki +sbu I L Stod Bhoti +sbv I A Sabine +sbw I L Simba +sbx I L Seberuang +sby I L Soli +sbz I L Sara Kaba +scb I L Chut +sce I L Dongxiang +scf I L San Miguel Creole French +scg I L Sanggau +sch I L Sakachep +sci I L Sri Lankan Creole Malay +sck I L Sadri +scl I L Shina +scn scn scn I L Sicilian +sco sco sco I L Scots +scp I L Helambu Sherpa +scq I L Sa'och +scs I L North Slavey +scu I L Shumcho +scv I L Sheni +scw I L Sha +scx I A Sicel +sda I L Toraja-Sa'dan +sdb I L Shabak +sdc I L Sassarese Sardinian +sde I L Surubu +sdf I L Sarli +sdg I L Savi +sdh I L Southern Kurdish +sdj I L Suundi +sdk I L Sos Kundi +sdl I L Saudi Arabian Sign Language +sdm I L Semandang +sdn I L Gallurese Sardinian +sdo I L Bukar-Sadung Bidayuh +sdp I L Sherdukpen +sdr I L Oraon Sadri +sds I E Sened +sdt I E Shuadit +sdu I L Sarudu +sdx I L Sibu Melanau +sdz I L Sallands +sea I L Semai +seb I L Shempire Senoufo +sec I L Sechelt +sed I L Sedang +see I L Seneca +sef I L Cebaara Senoufo +seg I L Segeju +seh I L Sena +sei I L Seri +sej I L Sene +sek I L Sekani +sel sel sel I L Selkup +sen I L Nanerigé Sénoufo +seo I L Suarmin +sep I L Sìcìté Sénoufo +seq I L Senara Sénoufo +ser I L Serrano +ses I L Koyraboro Senni Songhai +set I L Sentani +seu I L Serui-Laut +sev I L Nyarafolo Senoufo +sew I L Sewa Bay +sey I L Secoya +sez I L Senthang Chin +sfb I L Langue des signes de Belgique Francophone +sfe I L Eastern Subanen +sfm I L Small Flowery Miao +sfs I L South African Sign Language +sfw I L Sehwi +sga sga sga I H Old Irish (to 900) +sgb I L Mag-antsi Ayta +sgc I L Kipsigis +sgd I L Surigaonon +sge I L Segai +sgg I L Swiss-German Sign Language +sgh I L Shughni +sgi I L Suga +sgj I L Surgujia +sgk I L Sangkong +sgm I E Singa +sgo I L Songa +sgp I L Singpho +sgr I L Sangisari +sgs I L Samogitian +sgt I L Brokpake +sgu I L Salas +sgw I L Sebat Bet Gurage +sgx I L Sierra Leone Sign Language +sgy I L Sanglechi +sgz I L Sursurunga +sha I L Shall-Zwall +shb I L Ninam +shc I L Sonde +shd I L Kundal Shahi +she I L Sheko +shg I L Shua +shh I L Shoshoni +shi I L Tachelhit +shj I L Shatt +shk I L Shilluk +shl I L Shendu +shm I L Shahrudi +shn shn shn I L Shan +sho I L Shanga +shp I L Shipibo-Conibo +shq I L Sala +shr I L Shi +shs I L Shuswap +sht I E Shasta +shu I L Chadian Arabic +shv I L Shehri +shw I L Shwai +shx I L She +shy I L Tachawit +shz I L Syenara Senoufo +sia I E Akkala Sami +sib I L Sebop +sid sid sid I L Sidamo +sie I L Simaa +sif I L Siamou +sig I L Paasaal +sih I L Zire +sii I L Shom Peng +sij I L Numbami +sik I L Sikiana +sil I L Tumulung Sisaala +sim I L Mende (Papua New Guinea) +sin sin sin si I L Sinhala +sip I L Sikkimese +siq I L Sonia +sir I L Siri +sis I E Siuslaw +siu I L Sinagen +siv I L Sumariup +siw I L Siwai +six I L Sumau +siy I L Sivandi +siz I L Siwi +sja I L Epena +sjb I L Sajau Basap +sjd I L Kildin Sami +sje I L Pite Sami +sjg I L Assangori +sjk I E Kemi Sami +sjl I L Sajalong +sjm I L Mapun +sjn I C Sindarin +sjo I L Xibe +sjp I L Surjapuri +sjr I L Siar-Lak +sjs I E Senhaja De Srair +sjt I L Ter Sami +sju I L Ume Sami +sjw I L Shawnee +ska I L Skagit +skb I L Saek +skc I L Ma Manda +skd I L Southern Sierra Miwok +ske I L Seke (Vanuatu) +skf I L Sakirabiá +skg I L Sakalava Malagasy +skh I L Sikule +ski I L Sika +skj I L Seke (Nepal) +skk I L Sok +skm I L Kutong +skn I L Kolibugan Subanon +sko I L Seko Tengah +skp I L Sekapan +skq I L Sininkere +skr I L Seraiki +sks I L Maia +skt I L Sakata +sku I L Sakao +skv I L Skou +skw I E Skepi Creole Dutch +skx I L Seko Padang +sky I L Sikaiana +skz I L Sekar +slc I L Sáliba +sld I L Sissala +sle I L Sholaga +slf I L Swiss-Italian Sign Language +slg I L Selungai Murut +slh I L Southern Puget Sound Salish +sli I L Lower Silesian +slj I L Salumá +slk slo slk sk I L Slovak +sll I L Salt-Yui +slm I L Pangutaran Sama +sln I E Salinan +slp I L Lamaholot +slq I L Salchuq +slr I L Salar +sls I L Singapore Sign Language +slt I L Sila +slu I L Selaru +slv slv slv sl I L Slovenian +slw I L Sialum +slx I L Salampasu +sly I L Selayar +slz I L Ma'ya +sma sma sma I L Southern Sami +smb I L Simbari +smc I E Som +smd I L Sama +sme sme sme se I L Northern Sami +smf I L Auwe +smg I L Simbali +smh I L Samei +smj smj smj I L Lule Sami +smk I L Bolinao +sml I L Central Sama +smm I L Musasa +smn smn smn I L Inari Sami +smo smo smo sm I L Samoan +smp I E Samaritan +smq I L Samo +smr I L Simeulue +sms sms sms I L Skolt Sami +smt I L Simte +smu I E Somray +smv I L Samvedi +smw I L Sumbawa +smx I L Samba +smy I L Semnani +smz I L Simeku +sna sna sna sn I L Shona +snb I L Sebuyau +snc I L Sinaugoro +snd snd snd sd I L Sindhi +sne I L Bau Bidayuh +snf I L Noon +sng I L Sanga (Democratic Republic of Congo) +snh I E Shinabo +sni I E Sensi +snj I L Riverain Sango +snk snk snk I L Soninke +snl I L Sangil +snm I L Southern Ma'di +snn I L Siona +sno I L Snohomish +snp I L Siane +snq I L Sangu (Gabon) +snr I L Sihan +sns I L South West Bay +snu I L Senggi +snv I L Sa'ban +snw I L Selee +snx I L Sam +sny I L Saniyo-Hiyewe +snz I L Sinsauru +soa I L Thai Song +sob I L Sobei +soc I L So (Democratic Republic of Congo) +sod I L Songoora +soe I L Songomeno +sog sog sog I A Sogdian +soh I L Aka +soi I L Sonha +soj I L Soi +sok I L Sokoro +sol I L Solos +som som som so I L Somali +soo I L Songo +sop I L Songe +soq I L Kanasi +sor I L Somrai +sos I L Seeku +sot sot sot st I L Southern Sotho +sou I L Southern Thai +sov I L Sonsorol +sow I L Sowanda +sox I L Swo +soy I L Miyobe +soz I L Temi +spa spa spa es I L Spanish +spb I L Sepa (Indonesia) +spc I L Sapé +spd I L Saep +spe I L Sepa (Papua New Guinea) +spg I L Sian +spi I L Saponi +spk I L Sengo +spl I L Selepet +spm I L Akukem +spo I L Spokane +spp I L Supyire Senoufo +spq I L Loreto-Ucayali Spanish +spr I L Saparua +sps I L Saposa +spt I L Spiti Bhoti +spu I L Sapuan +spv I L Sambalpuri +spx I A South Picene +spy I L Sabaot +sqa I L Shama-Sambuga +sqh I L Shau +sqi alb sqi sq M L Albanian +sqk I L Albanian Sign Language +sqm I L Suma +sqn I E Susquehannock +sqo I L Sorkhei +sqq I L Sou +sqr I H Siculo Arabic +sqs I L Sri Lankan Sign Language +sqt I L Soqotri +squ I L Squamish +sra I L Saruga +srb I L Sora +src I L Logudorese Sardinian +srd srd srd sc M L Sardinian +sre I L Sara +srf I L Nafi +srg I L Sulod +srh I L Sarikoli +sri I L Siriano +srk I L Serudung Murut +srl I L Isirawa +srm I L Saramaccan +srn srn srn I L Sranan Tongo +sro I L Campidanese Sardinian +srp srp srp sr I L Serbian +srq I L Sirionó +srr srr srr I L Serer +srs I L Sarsi +srt I L Sauri +sru I L Suruí +srv I L Southern Sorsoganon +srw I L Serua +srx I L Sirmauri +sry I L Sera +srz I L Shahmirzadi +ssb I L Southern Sama +ssc I L Suba-Simbiti +ssd I L Siroi +sse I L Balangingi +ssf I L Thao +ssg I L Seimat +ssh I L Shihhi Arabic +ssi I L Sansi +ssj I L Sausi +ssk I L Sunam +ssl I L Western Sisaala +ssm I L Semnam +ssn I L Waata +sso I L Sissano +ssp I L Spanish Sign Language +ssq I L So'a +ssr I L Swiss-French Sign Language +sss I L Sô +sst I L Sinasina +ssu I L Susuami +ssv I L Shark Bay +ssw ssw ssw ss I L Swati +ssx I L Samberigi +ssy I L Saho +ssz I L Sengseng +sta I L Settla +stb I L Northern Subanen +std I L Sentinel +ste I L Liana-Seti +stf I L Seta +stg I L Trieng +sth I L Shelta +sti I L Bulo Stieng +stj I L Matya Samo +stk I L Arammba +stl I L Stellingwerfs +stm I L Setaman +stn I L Owa +sto I L Stoney +stp I L Southeastern Tepehuan +stq I L Saterfriesisch +str I L Straits Salish +sts I L Shumashti +stt I L Budeh Stieng +stu I L Samtao +stv I L Silt'e +stw I L Satawalese +sty I L Siberian Tatar +sua I L Sulka +sub I L Suku +suc I L Western Subanon +sue I L Suena +sug I L Suganga +sui I L Suki +suj I L Shubi +suk suk suk I L Sukuma +sun sun sun su I L Sundanese +suq I L Suri +sur I L Mwaghavul +sus sus sus I L Susu +sut I E Subtiaba +suv I L Puroik +suw I L Sumbwa +sux sux sux I A Sumerian +suy I L Suyá +suz I L Sunwar +sva I L Svan +svb I L Ulau-Suain +svc I L Vincentian Creole English +sve I L Serili +svk I L Slovakian Sign Language +svm I L Slavomolisano +svr I L Savara +svs I L Savosavo +svx I E Skalvian +swa swa swa sw M L Swahili (macrolanguage) +swb I L Maore Comorian +swc I L Congo Swahili +swe swe swe sv I L Swedish +swf I L Sere +swg I L Swabian +swh I L Swahili (individual language) +swi I L Sui +swj I L Sira +swk I L Malawi Sena +swl I L Swedish Sign Language +swm I L Samosa +swn I L Sawknah +swo I L Shanenawa +swp I L Suau +swq I L Sharwa +swr I L Saweru +sws I L Seluwasan +swt I L Sawila +swu I L Suwawa +swv I L Shekhawati +sww I E Sowa +swx I L Suruahá +swy I L Sarua +sxb I L Suba +sxc I A Sicanian +sxe I L Sighu +sxg I L Shixing +sxk I E Southern Kalapuya +sxl I E Selian +sxm I L Samre +sxn I L Sangir +sxo I A Sorothaptic +sxr I L Saaroa +sxs I L Sasaru +sxu I L Upper Saxon +sxw I L Saxwe Gbe +sya I L Siang +syb I L Central Subanen +syc syc syc I H Classical Syriac +syi I L Seki +syk I L Sukur +syl I L Sylheti +sym I L Maya Samo +syn I L Senaya +syo I L Suoy +syr syr syr M L Syriac +sys I L Sinyar +syw I L Kagate +syy I L Al-Sayyid Bedouin Sign Language +sza I L Semelai +szb I L Ngalum +szc I L Semaq Beri +szd I E Seru +sze I L Seze +szg I L Sengele +szl I L Silesian +szn I L Sula +szp I L Suabo +szv I L Isu (Fako Division) +szw I L Sawai +taa I L Lower Tanana +tab I L Tabassaran +tac I L Lowland Tarahumara +tad I L Tause +tae I L Tariana +taf I L Tapirapé +tag I L Tagoi +tah tah tah ty I L Tahitian +taj I L Eastern Tamang +tak I L Tala +tal I L Tal +tam tam tam ta I L Tamil +tan I L Tangale +tao I L Yami +tap I L Taabwa +taq I L Tamasheq +tar I L Central Tarahumara +tas I E Tay Boi +tat tat tat tt I L Tatar +tau I L Upper Tanana +tav I L Tatuyo +taw I L Tai +tax I L Tamki +tay I L Atayal +taz I L Tocho +tba I L Aikanã +tbb I E Tapeba +tbc I L Takia +tbd I L Kaki Ae +tbe I L Tanimbili +tbf I L Mandara +tbg I L North Tairora +tbh I E Thurawal +tbi I L Gaam +tbj I L Tiang +tbk I L Calamian Tagbanwa +tbl I L Tboli +tbm I L Tagbu +tbn I L Barro Negro Tunebo +tbo I L Tawala +tbp I L Taworta +tbr I L Tumtum +tbs I L Tanguat +tbt I L Tembo (Kitembo) +tbu I E Tubar +tbv I L Tobo +tbw I L Tagbanwa +tbx I L Kapin +tby I L Tabaru +tbz I L Ditammari +tca I L Ticuna +tcb I L Tanacross +tcc I L Datooga +tcd I L Tafi +tce I L Southern Tutchone +tcf I L Malinaltepec Me'phaa +tcg I L Tamagario +tch I L Turks And Caicos Creole English +tci I L Wára +tck I L Tchitchege +tcl I E Taman (Myanmar) +tcm I L Tanahmerah +tcn I L Tichurong +tco I L Taungyo +tcp I L Tawr Chin +tcq I L Kaiy +tcs I L Torres Strait Creole +tct I L T'en +tcu I L Southeastern Tarahumara +tcw I L Tecpatlán Totonac +tcx I L Toda +tcy I L Tulu +tcz I L Thado Chin +tda I L Tagdal +tdb I L Panchpargania +tdc I L Emberá-Tadó +tdd I L Tai Nüa +tde I L Tiranige Diga Dogon +tdf I L Talieng +tdg I L Western Tamang +tdh I L Thulung +tdi I L Tomadino +tdj I L Tajio +tdk I L Tambas +tdl I L Sur +tdn I L Tondano +tdo I L Teme +tdq I L Tita +tdr I L Todrah +tds I L Doutai +tdt I L Tetun Dili +tdu I L Tempasuk Dusun +tdv I L Toro +tdx I L Tandroy-Mahafaly Malagasy +tdy I L Tadyawan +tea I L Temiar +teb I E Tetete +tec I L Terik +ted I L Tepo Krumen +tee I L Huehuetla Tepehua +tef I L Teressa +teg I L Teke-Tege +teh I L Tehuelche +tei I L Torricelli +tek I L Ibali Teke +tel tel tel te I L Telugu +tem tem tem I L Timne +ten I E Tama (Colombia) +teo I L Teso +tep I E Tepecano +teq I L Temein +ter ter ter I L Tereno +tes I L Tengger +tet tet tet I L Tetum +teu I L Soo +tev I L Teor +tew I L Tewa (USA) +tex I L Tennet +tey I L Tulishi +tfi I L Tofin Gbe +tfn I L Tanaina +tfo I L Tefaro +tfr I L Teribe +tft I L Ternate +tga I L Sagalla +tgb I L Tobilung +tgc I L Tigak +tgd I L Ciwogai +tge I L Eastern Gorkha Tamang +tgf I L Chalikha +tgh I L Tobagonian Creole English +tgi I L Lawunuia +tgj I L Tagin +tgk tgk tgk tg I L Tajik +tgl tgl tgl tl I L Tagalog +tgn I L Tandaganon +tgo I L Sudest +tgp I L Tangoa +tgq I L Tring +tgr I L Tareng +tgs I L Nume +tgt I L Central Tagbanwa +tgu I L Tanggu +tgv I E Tingui-Boto +tgw I L Tagwana Senoufo +tgx I L Tagish +tgy I E Togoyo +tgz I E Tagalaka +tha tha tha th I L Thai +thc I L Tai Hang Tong +thd I L Thayore +the I L Chitwania Tharu +thf I L Thangmi +thh I L Northern Tarahumara +thi I L Tai Long +thk I L Tharaka +thl I L Dangaura Tharu +thm I L Aheu +thn I L Thachanadan +thp I L Thompson +thq I L Kochila Tharu +thr I L Rana Tharu +ths I L Thakali +tht I L Tahltan +thu I L Thuri +thv I L Tahaggart Tamahaq +thw I L Thudam +thx I L The +thy I L Tha +thz I L Tayart Tamajeq +tia I L Tidikelt Tamazight +tic I L Tira +tid I L Tidong +tif I L Tifal +tig tig tig I L Tigre +tih I L Timugon Murut +tii I L Tiene +tij I L Tilung +tik I L Tikar +til I E Tillamook +tim I L Timbe +tin I L Tindi +tio I L Teop +tip I L Trimuris +tiq I L Tiéfo +tir tir tir ti I L Tigrinya +tis I L Masadiit Itneg +tit I L Tinigua +tiu I L Adasen +tiv tiv tiv I L Tiv +tiw I L Tiwi +tix I L Southern Tiwa +tiy I L Tiruray +tiz I L Tai Hongjin +tja I L Tajuasohn +tjg I L Tunjung +tji I L Northern Tujia +tjl I L Tai Laing +tjm I E Timucua +tjn I E Tonjon +tjo I L Temacine Tamazight +tjs I L Southern Tujia +tju I E Tjurruru +tjw I L Djabwurrung +tka I E Truká +tkb I L Buksa +tkd I L Tukudede +tke I L Takwane +tkf I E Tukumanféd +tkg I L Tesaka Malagasy +tkl tkl tkl I L Tokelau +tkm I E Takelma +tkn I L Toku-No-Shima +tkp I L Tikopia +tkq I L Tee +tkr I L Tsakhur +tks I L Takestani +tkt I L Kathoriya Tharu +tku I L Upper Necaxa Totonac +tkw I L Teanu +tkx I L Tangko +tkz I L Takua +tla I L Southwestern Tepehuan +tlb I L Tobelo +tlc I L Yecuatla Totonac +tld I L Talaud +tlf I L Telefol +tlg I L Tofanma +tlh tlh tlh I C Klingon +tli tli tli I L Tlingit +tlj I L Talinga-Bwisi +tlk I L Taloki +tll I L Tetela +tlm I L Tolomako +tln I L Talondo' +tlo I L Talodi +tlp I L Filomena Mata-Coahuitlán Totonac +tlq I L Tai Loi +tlr I L Talise +tls I L Tambotalo +tlt I L Teluti +tlu I L Tulehu +tlv I L Taliabu +tlx I L Khehek +tly I L Talysh +tma I L Tama (Chad) +tmb I L Katbol +tmc I L Tumak +tmd I L Haruai +tme I E Tremembé +tmf I L Toba-Maskoy +tmg I E Ternateño +tmh tmh tmh M L Tamashek +tmi I L Tutuba +tmj I L Samarokena +tmk I L Northwestern Tamang +tml I L Tamnim Citak +tmm I L Tai Thanh +tmn I L Taman (Indonesia) +tmo I L Temoq +tmp I L Tai Mène +tmq I L Tumleo +tmr I E Jewish Babylonian Aramaic (ca. 200-1200 CE) +tms I L Tima +tmt I L Tasmate +tmu I L Iau +tmv I L Tembo (Motembo) +tmw I L Temuan +tmy I L Tami +tmz I E Tamanaku +tna I L Tacana +tnb I L Western Tunebo +tnc I L Tanimuca-Retuarã +tnd I L Angosturas Tunebo +tne I L Tinoc Kallahan +tng I L Tobanga +tnh I L Maiani +tni I L Tandia +tnk I L Kwamera +tnl I L Lenakel +tnm I L Tabla +tnn I L North Tanna +tno I L Toromono +tnp I L Whitesands +tnq I E Taino +tnr I L Ménik +tns I L Tenis +tnt I L Tontemboan +tnu I L Tay Khang +tnv I L Tangchangya +tnw I L Tonsawang +tnx I L Tanema +tny I L Tongwe +tnz I L Tonga (Thailand) +tob I L Toba +toc I L Coyutla Totonac +tod I L Toma +toe I E Tomedes +tof I L Gizrra +tog tog tog I L Tonga (Nyasa) +toh I L Gitonga +toi I L Tonga (Zambia) +toj I L Tojolabal +tol I L Tolowa +tom I L Tombulu +ton ton ton to I L Tonga (Tonga Islands) +too I L Xicotepec De Juárez Totonac +top I L Papantla Totonac +toq I L Toposa +tor I L Togbo-Vara Banda +tos I L Highland Totonac +tou I L Tho +tov I L Upper Taromi +tow I L Jemez +tox I L Tobian +toy I L Topoiyo +toz I L To +tpa I L Taupota +tpc I L Azoyú Me'phaa +tpe I L Tippera +tpf I L Tarpia +tpg I L Kula +tpi tpi tpi I L Tok Pisin +tpj I L Tapieté +tpk I E Tupinikin +tpl I L Tlacoapa Me'phaa +tpm I L Tampulma +tpn I E Tupinambá +tpo I L Tai Pao +tpp I L Pisaflores Tepehua +tpq I L Tukpa +tpr I L Tuparí +tpt I L Tlachichilco Tepehua +tpu I L Tampuan +tpv I L Tanapag +tpw I E Tupí +tpx I L Acatepec Me'phaa +tpy I L Trumai +tpz I L Tinputz +tqb I L Tembé +tql I L Lehali +tqm I L Turumsa +tqn I L Tenino +tqo I L Toaripi +tqp I L Tomoip +tqq I L Tunni +tqr I E Torona +tqt I L Western Totonac +tqu I L Touo +tqw I E Tonkawa +tra I L Tirahi +trb I L Terebu +trc I L Copala Triqui +trd I L Turi +tre I L East Tarangan +trf I L Trinidadian Creole English +trg I L Lishán Didán +trh I L Turaka +tri I L Trió +trj I L Toram +trl I L Traveller Scottish +trm I L Tregami +trn I L Trinitario +tro I L Tarao Naga +trp I L Kok Borok +trq I L San Martín Itunyoso Triqui +trr I L Taushiro +trs I L Chicahuaxtla Triqui +trt I L Tunggare +tru I L Turoyo +trv I L Taroko +trw I L Torwali +trx I L Tringgus-Sembaan Bidayuh +try I E Turung +trz I E Torá +tsa I L Tsaangi +tsb I L Tsamai +tsc I L Tswa +tsd I L Tsakonian +tse I L Tunisian Sign Language +tsf I L Southwestern Tamang +tsg I L Tausug +tsh I L Tsuvan +tsi tsi tsi I L Tsimshian +tsj I L Tshangla +tsk I L Tseku +tsl I L Ts'ün-Lao +tsm I L Turkish Sign Language +tsn tsn tsn tn I L Tswana +tso tso tso ts I L Tsonga +tsp I L Northern Toussian +tsq I L Thai Sign Language +tsr I L Akei +tss I L Taiwan Sign Language +tst I L Tondi Songway Kiini +tsu I L Tsou +tsv I L Tsogo +tsw I L Tsishingini +tsx I L Mubami +tsy I L Tebul Sign Language +tsz I L Purepecha +tta I E Tutelo +ttb I L Gaa +ttc I L Tektiteko +ttd I L Tauade +tte I L Bwanabwana +ttf I L Tuotomb +ttg I L Tutong +tth I L Upper Ta'oih +tti I L Tobati +ttj I L Tooro +ttk I L Totoro +ttl I L Totela +ttm I L Northern Tutchone +ttn I L Towei +tto I L Lower Ta'oih +ttp I L Tombelala +ttq I L Tawallammat Tamajaq +ttr I L Tera +tts I L Northeastern Thai +ttt I L Muslim Tat +ttu I L Torau +ttv I L Titan +ttw I L Long Wat +tty I L Sikaritai +ttz I L Tsum +tua I L Wiarumus +tub I L Tübatulabal +tuc I L Mutu +tud I E Tuxá +tue I L Tuyuca +tuf I L Central Tunebo +tug I L Tunia +tuh I L Taulil +tui I L Tupuri +tuj I L Tugutil +tuk tuk tuk tk I L Turkmen +tul I L Tula +tum tum tum I L Tumbuka +tun I E Tunica +tuo I L Tucano +tuq I L Tedaga +tur tur tur tr I L Turkish +tus I L Tuscarora +tuu I L Tututni +tuv I L Turkana +tux I E Tuxináwa +tuy I L Tugen +tuz I L Turka +tva I L Vaghua +tvd I L Tsuvadi +tve I L Te'un +tvk I L Southeast Ambrym +tvl tvl tvl I L Tuvalu +tvm I L Tela-Masbuar +tvn I L Tavoyan +tvo I L Tidore +tvs I L Taveta +tvt I L Tutsa Naga +tvu I L Tunen +tvw I L Sedoa +tvy I E Timor Pidgin +twa I E Twana +twb I L Western Tawbuid +twc I E Teshenawa +twd I L Twents +twe I L Tewa (Indonesia) +twf I L Northern Tiwa +twg I L Tereweng +twh I L Tai Dón +twi twi twi tw I L Twi +twl I L Tawara +twm I L Tawang Monpa +twn I L Twendi +two I L Tswapong +twp I L Ere +twq I L Tasawaq +twr I L Southwestern Tarahumara +twt I E Turiwára +twu I L Termanu +tww I L Tuwari +twx I L Tewe +twy I L Tawoyan +txa I L Tombonuo +txb I A Tokharian B +txc I E Tsetsaut +txe I L Totoli +txg I A Tangut +txh I A Thracian +txi I L Ikpeng +txm I L Tomini +txn I L West Tarangan +txo I L Toto +txq I L Tii +txr I A Tartessian +txs I L Tonsea +txt I L Citak +txu I L Kayapó +txx I L Tatana +txy I L Tanosy Malagasy +tya I L Tauya +tye I L Kyanga +tyh I L O'du +tyi I L Teke-Tsaayi +tyj I L Tai Do +tyl I L Thu Lao +tyn I L Kombai +typ I E Thaypan +tyr I L Tai Daeng +tys I L Tày Sa Pa +tyt I L Tày Tac +tyu I L Kua +tyv tyv tyv I L Tuvinian +tyx I L Teke-Tyee +tyz I L Tày +tza I L Tanzanian Sign Language +tzh I L Tzeltal +tzj I L Tz'utujil +tzl I C Talossan +tzm I L Central Atlas Tamazight +tzn I L Tugun +tzo I L Tzotzil +tzx I L Tabriak +uam I E Uamué +uan I L Kuan +uar I L Tairuma +uba I L Ubang +ubi I L Ubi +ubl I L Buhi'non Bikol +ubr I L Ubir +ubu I L Umbu-Ungu +uby I E Ubykh +uda I L Uda +ude I L Udihe +udg I L Muduga +udi I L Udi +udj I L Ujir +udl I L Wuzlam +udm udm udm I L Udmurt +udu I L Uduk +ues I L Kioko +ufi I L Ufim +uga uga uga I A Ugaritic +ugb I E Kuku-Ugbanh +uge I L Ughele +ugn I L Ugandan Sign Language +ugo I L Ugong +ugy I L Uruguayan Sign Language +uha I L Uhami +uhn I L Damal +uig uig uig ug I L Uighur +uis I L Uisai +uiv I L Iyive +uji I L Tanjijili +uka I L Kaburi +ukg I L Ukuriguma +ukh I L Ukhwejo +ukl I L Ukrainian Sign Language +ukp I L Ukpe-Bayobiri +ukq I L Ukwa +ukr ukr ukr uk I L Ukrainian +uks I L Urubú-Kaapor Sign Language +uku I L Ukue +ukw I L Ukwuani-Aboh-Ndoni +uky I E Kuuk-Yak +ula I L Fungwa +ulb I L Ulukwumi +ulc I L Ulch +ule I E Lule +ulf I L Usku +uli I L Ulithian +ulk I L Meriam +ull I L Ullatan +ulm I L Ulumanda' +uln I L Unserdeutsch +ulu I L Uma' Lung +ulw I L Ulwa +uma I L Umatilla +umb umb umb I L Umbundu +umc I A Marrucinian +umd I E Umbindhamu +umg I E Umbuygamu +umi I L Ukit +umm I L Umon +umn I L Makyan Naga +umo I E Umotína +ump I L Umpila +umr I E Umbugarla +ums I L Pendau +umu I L Munsee +una I L North Watut +und und und S S Undetermined +une I L Uneme +ung I L Ngarinyin +unk I L Enawené-Nawé +unm I E Unami +unn I L Kurnai +unr I L Mundari +unu I L Unubahe +unx I L Munda +unz I L Unde Kaili +uok I L Uokha +upi I L Umeda +upv I L Uripiv-Wala-Rano-Atchin +ura I L Urarina +urb I L Urubú-Kaapor +urc I E Urningangg +urd urd urd ur I L Urdu +ure I L Uru +urf I E Uradhi +urg I L Urigina +urh I L Urhobo +uri I L Urim +urk I L Urak Lawoi' +url I L Urali +urm I L Urapmin +urn I L Uruangnirin +uro I L Ura (Papua New Guinea) +urp I L Uru-Pa-In +urr I L Lehalurup +urt I L Urat +uru I E Urumi +urv I E Uruava +urw I L Sop +urx I L Urimo +ury I L Orya +urz I L Uru-Eu-Wau-Wau +usa I L Usarufa +ush I L Ushojo +usi I L Usui +usk I L Usaghade +usp I L Uspanteco +usu I L Uya +uta I L Otank +ute I L Ute-Southern Paiute +utp I L Amba (Solomon Islands) +utr I L Etulo +utu I L Utu +uum I L Urum +uun I L Kulon-Pazeh +uur I L Ura (Vanuatu) +uuu I L U +uve I L West Uvean +uvh I L Uri +uvl I L Lote +uwa I L Kuku-Uwanh +uya I L Doko-Uyanga +uzb uzb uzb uz M L Uzbek +uzn I L Northern Uzbek +uzs I L Southern Uzbek +vaa I L Vaagri Booli +vae I L Vale +vaf I L Vafsi +vag I L Vagla +vah I L Varhadi-Nagpuri +vai vai vai I L Vai +vaj I L Vasekela Bushman +val I L Vehes +vam I L Vanimo +van I L Valman +vao I L Vao +vap I L Vaiphei +var I L Huarijio +vas I L Vasavi +vau I L Vanuma +vav I L Varli +vay I L Wayu +vbb I L Southeast Babar +vbk I L Southwestern Bontok +vec I L Venetian +ved I L Veddah +vel I L Veluws +vem I L Vemgo-Mabas +ven ven ven ve I L Venda +veo I E Ventureño +vep I L Veps +ver I L Mom Jango +vgr I L Vaghri +vgt I L Vlaamse Gebarentaal +vic I L Virgin Islands Creole English +vid I L Vidunda +vie vie vie vi I L Vietnamese +vif I L Vili +vig I L Viemo +vil I L Vilela +vin I L Vinza +vis I L Vishavan +vit I L Viti +viv I L Iduna +vka I E Kariyarra +vki I L Ija-Zuba +vkj I L Kujarge +vkk I L Kaur +vkl I L Kulisusu +vkm I E Kamakan +vko I L Kodeoha +vkp I L Korlai Creole Portuguese +vkt I L Tenggarong Kutai Malay +vku I L Kurrama +vlp I L Valpei +vls I L Vlaams +vma I L Martuyhunira +vmb I E Barbaram +vmc I L Juxtlahuaca Mixtec +vmd I L Mudu Koraga +vme I L East Masela +vmf I L Mainfränkisch +vmg I L Lungalunga +vmh I L Maraghei +vmi I E Miwa +vmj I L Ixtayutla Mixtec +vmk I L Makhuwa-Shirima +vml I E Malgana +vmm I L Mitlatongo Mixtec +vmp I L Soyaltepec Mazatec +vmq I L Soyaltepec Mixtec +vmr I L Marenje +vms I E Moksela +vmu I E Muluridyi +vmv I E Valley Maidu +vmw I L Makhuwa +vmx I L Tamazola Mixtec +vmy I L Ayautla Mazatec +vmz I L Mazatlán Mazatec +vnk I L Vano +vnm I L Vinmavis +vnp I L Vunapu +vol vol vol vo I C Volapük +vor I L Voro +vot vot vot I L Votic +vra I L Vera'a +vro I L Võro +vrs I L Varisi +vrt I L Burmbar +vsi I L Moldova Sign Language +vsl I L Venezuelan Sign Language +vsv I L Valencian Sign Language +vto I L Vitou +vum I L Vumbu +vun I L Vunjo +vut I L Vute +vwa I L Awa (China) +waa I L Walla Walla +wab I L Wab +wac I L Wasco-Wishram +wad I L Wandamen +wae I L Walser +waf I E Wakoná +wag I L Wa'ema +wah I L Watubela +wai I L Wares +waj I L Waffa +wal wal wal I L Wolaytta +wam I E Wampanoag +wan I L Wan +wao I E Wappo +wap I L Wapishana +waq I L Wageman +war war war I L Waray (Philippines) +was was was I L Washo +wat I L Kaninuwa +wau I L Waurá +wav I L Waka +waw I L Waiwai +wax I L Watam +way I L Wayana +waz I L Wampur +wba I L Warao +wbb I L Wabo +wbe I L Waritai +wbf I L Wara +wbh I L Wanda +wbi I L Vwanji +wbj I L Alagwa +wbk I L Waigali +wbl I L Wakhi +wbm I L Wa +wbp I L Warlpiri +wbq I L Waddar +wbr I L Wagdi +wbt I L Wanman +wbv I L Wajarri +wbw I L Woi +wca I L Yanomámi +wci I L Waci Gbe +wdd I L Wandji +wdg I L Wadaginam +wdj I L Wadjiginy +wdk I E Wadikali +wdu I E Wadjigu +wdy I E Wadjabangayi +wea I E Wewaw +wec I L Wè Western +wed I L Wedau +weg I L Wergaia +weh I L Weh +wei I L Kiunum +wem I L Weme Gbe +weo I L Wemale +wep I L Westphalien +wer I L Weri +wes I L Cameroon Pidgin +wet I L Perai +weu I L Rawngtu Chin +wew I L Wejewa +wfg I L Yafi +wga I E Wagaya +wgb I L Wagawaga +wgg I E Wangganguru +wgi I L Wahgi +wgo I L Waigeo +wgu I E Wirangu +wgy I L Warrgamay +wha I L Manusela +whg I L North Wahgi +whk I L Wahau Kenyah +whu I L Wahau Kayan +wib I L Southern Toussian +wic I L Wichita +wie I E Wik-Epa +wif I E Wik-Keyangan +wig I L Wik-Ngathana +wih I L Wik-Me'anha +wii I L Minidien +wij I L Wik-Iiyanh +wik I L Wikalkan +wil I E Wilawila +wim I L Wik-Mungkan +win I L Ho-Chunk +wir I E Wiraféd +wiu I L Wiru +wiv I L Vitu +wiy I E Wiyot +wja I L Waja +wji I L Warji +wka I E Kw'adza +wkb I L Kumbaran +wkd I L Wakde +wkl I L Kalanadi +wku I L Kunduvadi +wkw I E Wakawaka +wky I E Wangkayutyuru +wla I L Walio +wlc I L Mwali Comorian +wle I L Wolane +wlg I L Kunbarlang +wli I L Waioli +wlk I E Wailaki +wll I L Wali (Sudan) +wlm I H Middle Welsh +wln wln wln wa I L Walloon +wlo I L Wolio +wlr I L Wailapa +wls I L Wallisian +wlu I E Wuliwuli +wlv I L Wichí Lhamtés Vejoz +wlw I L Walak +wlx I L Wali (Ghana) +wly I E Waling +wma I E Mawa (Nigeria) +wmb I L Wambaya +wmc I L Wamas +wmd I L Mamaindé +wme I L Wambule +wmh I L Waima'a +wmi I E Wamin +wmm I L Maiwa (Indonesia) +wmn I E Waamwang +wmo I L Wom (Papua New Guinea) +wms I L Wambon +wmt I L Walmajarri +wmw I L Mwani +wmx I L Womo +wnb I L Wanambre +wnc I L Wantoat +wnd I E Wandarang +wne I L Waneci +wng I L Wanggom +wni I L Ndzwani Comorian +wnk I L Wanukaka +wnm I E Wanggamala +wnn I E Wunumara +wno I L Wano +wnp I L Wanap +wnu I L Usan +wnw I L Wintu +wny I L Wanyi +woa I L Tyaraity +wob I L Wè Northern +woc I L Wogeo +wod I L Wolani +woe I L Woleaian +wof I L Gambian Wolof +wog I L Wogamusin +woi I L Kamang +wok I L Longto +wol wol wol wo I L Wolof +wom I L Wom (Nigeria) +won I L Wongo +woo I L Manombai +wor I L Woria +wos I L Hanga Hundi +wow I L Wawonii +woy I E Weyto +wpc I L Maco +wra I L Warapu +wrb I E Warluwara +wrd I L Warduji +wrg I E Warungu +wrh I E Wiradhuri +wri I E Wariyangga +wrk I L Garrwa +wrl I L Warlmanpa +wrm I L Warumungu +wrn I L Warnang +wro I E Worrorra +wrp I L Waropen +wrr I L Wardaman +wrs I L Waris +wru I L Waru +wrv I L Waruna +wrw I E Gugu Warra +wrx I L Wae Rana +wry I L Merwari +wrz I E Waray (Australia) +wsa I L Warembori +wsi I L Wusi +wsk I L Waskia +wsr I L Owenia +wss I L Wasa +wsu I E Wasu +wsv I E Wotapuri-Katarqalai +wtf I L Watiwa +wth I E Wathawurrung +wti I L Berta +wtk I L Watakataui +wtm I L Mewati +wtw I L Wotu +wua I L Wikngenchera +wub I L Wunambal +wud I L Wudu +wuh I L Wutunhua +wul I L Silimo +wum I L Wumbvu +wun I L Bungu +wur I E Wurrugu +wut I L Wutung +wuu I L Wu Chinese +wuv I L Wuvulu-Aua +wux I L Wulna +wuy I L Wauyai +wwa I L Waama +wwb I E Wakabunga +wwo I L Wetamut +wwr I E Warrwa +www I L Wawa +wxa I L Waxianghua +wxw I E Wardandi +wya I L Wyandot +wyb I L Wangaaybuwan-Ngiyambaa +wyi I E Woiwurrung +wym I L Wymysorys +wyr I L Wayoró +wyy I L Western Fijian +xaa I H Andalusian Arabic +xab I L Sambe +xac I L Kachari +xad I E Adai +xae I A Aequian +xag I E Aghwan +xai I E Kaimbé +xal xal xal I L Kalmyk +xam I E /Xam +xan I L Xamtanga +xao I L Khao +xap I E Apalachee +xaq I A Aquitanian +xar I E Karami +xas I E Kamas +xat I L Katawixi +xau I L Kauwera +xav I L Xavánte +xaw I L Kawaiisu +xay I L Kayan Mahakam +xba I E Kamba (Brazil) +xbb I E Lower Burdekin +xbc I A Bactrian +xbd I E Bindal +xbe I E Bigambal +xbg I E Bunganditj +xbi I L Kombio +xbj I E Birrpayi +xbm I H Middle Breton +xbn I E Kenaboi +xbo I E Bolgarian +xbp I E Bibbulman +xbr I L Kambera +xbw I E Kambiwá +xbx I E Kabixí +xby I L Batyala +xcb I E Cumbric +xcc I A Camunic +xce I A Celtiberian +xcg I A Cisalpine Gaulish +xch I E Chemakum +xcl I H Classical Armenian +xcm I E Comecrudo +xcn I E Cotoname +xco I A Chorasmian +xcr I A Carian +xct I H Classical Tibetan +xcu I E Curonian +xcv I E Chuvantsy +xcw I E Coahuilteco +xcy I E Cayuse +xda I L Darkinyung +xdc I A Dacian +xdk I E Dharuk +xdm I A Edomite +xdy I L Malayic Dayak +xeb I A Eblan +xed I L Hdi +xeg I E //Xegwi +xel I L Kelo +xem I L Kembayan +xep I A Epi-Olmec +xer I L Xerénte +xes I L Kesawai +xet I L Xetá +xeu I L Keoru-Ahia +xfa I A Faliscan +xga I A Galatian +xgb I E Gbin +xgd I E Gudang +xgf I E Gabrielino-Fernandeño +xgg I E Goreng +xgi I E Garingbal +xgl I E Galindan +xgm I E Guwinmal +xgr I E Garza +xgu I L Unggumi +xgw I E Guwa +xha I A Harami +xhc I E Hunnic +xhd I A Hadrami +xhe I L Khetrani +xho xho xho xh I L Xhosa +xhr I A Hernican +xht I A Hattic +xhu I A Hurrian +xhv I L Khua +xib I A Iberian +xii I L Xiri +xil I A Illyrian +xin I E Xinca +xip I E Xipináwa +xir I E Xiriâna +xiv I A Indus Valley Language +xiy I L Xipaya +xjb I E Minjungbal +xjt I E Jaitmatang +xka I L Kalkoti +xkb I L Northern Nago +xkc I L Kho'ini +xkd I L Mendalam Kayan +xke I L Kereho +xkf I L Khengkha +xkg I L Kagoro +xkh I L Karahawyana +xki I L Kenyan Sign Language +xkj I L Kajali +xkk I L Kaco' +xkl I L Mainstream Kenyah +xkn I L Kayan River Kayan +xko I L Kiorr +xkp I L Kabatei +xkq I L Koroni +xkr I E Xakriabá +xks I L Kumbewaha +xkt I L Kantosi +xku I L Kaamba +xkv I L Kgalagadi +xkw I L Kembra +xkx I L Karore +xky I L Uma' Lasan +xkz I L Kurtokha +xla I L Kamula +xlb I E Loup B +xlc I A Lycian +xld I A Lydian +xle I A Lemnian +xlg I A Ligurian (Ancient) +xli I A Liburnian +xln I A Alanic +xlo I E Loup A +xlp I A Lepontic +xls I A Lusitanian +xlu I A Cuneiform Luwian +xly I A Elymian +xma I L Mushungulu +xmb I L Mbonga +xmc I L Makhuwa-Marrevone +xmd I L Mbudum +xme I A Median +xmf I L Mingrelian +xmg I L Mengaka +xmh I L Kuku-Muminh +xmj I L Majera +xmk I A Ancient Macedonian +xml I L Malaysian Sign Language +xmm I L Manado Malay +xmn I H Manichaean Middle Persian +xmo I L Morerebi +xmp I E Kuku-Mu'inh +xmq I E Kuku-Mangk +xmr I A Meroitic +xms I L Moroccan Sign Language +xmt I L Matbat +xmu I E Kamu +xmv I L Antankarana Malagasy +xmw I L Tsimihety Malagasy +xmx I L Maden +xmy I L Mayaguduna +xmz I L Mori Bawah +xna I A Ancient North Arabian +xnb I L Kanakanabu +xng I H Middle Mongolian +xnh I L Kuanhua +xni I E Ngarigu +xnk I E Nganakarti +xnn I L Northern Kankanay +xno I H Anglo-Norman +xnr I L Kangri +xns I L Kanashi +xnt I E Narragansett +xnu I E Nukunul +xny I L Nyiyaparli +xnz I L Kenzi +xoc I E O'chi'chi' +xod I L Kokoda +xog I L Soga +xoi I L Kominimung +xok I L Xokleng +xom I L Komo (Sudan) +xon I L Konkomba +xoo I E Xukurú +xop I L Kopar +xor I L Korubo +xow I L Kowaki +xpa I E Pirriya +xpc I E Pecheneg +xpe I L Liberia Kpelle +xpg I A Phrygian +xpi I E Pictish +xpj I E Mpalitjanh +xpk I L Kulina Pano +xpm I E Pumpokol +xpn I E Kapinawá +xpo I E Pochutec +xpp I E Puyo-Paekche +xpq I E Mohegan-Pequot +xpr I A Parthian +xps I E Pisidian +xpt I E Punthamara +xpu I A Punic +xpy I E Puyo +xqa I H Karakhanid +xqt I A Qatabanian +xra I L Krahô +xrb I L Eastern Karaboro +xrd I E Gundungurra +xre I L Kreye +xrg I E Minang +xri I L Krikati-Timbira +xrm I E Armazic +xrn I E Arin +xrq I E Karranga +xrr I A Raetic +xrt I E Aranama-Tamique +xru I L Marriammu +xrw I L Karawa +xsa I A Sabaean +xsb I L Sambal +xsc I A Scythian +xsd I A Sidetic +xse I L Sempan +xsh I L Shamang +xsi I L Sio +xsj I L Subi +xsl I L South Slavey +xsm I L Kasem +xsn I L Sanga (Nigeria) +xso I E Solano +xsp I L Silopi +xsq I L Makhuwa-Saka +xsr I L Sherpa +xss I E Assan +xsu I L Sanumá +xsv I E Sudovian +xsy I L Saisiyat +xta I L Alcozauca Mixtec +xtb I L Chazumba Mixtec +xtc I L Katcha-Kadugli-Miri +xtd I L Diuxi-Tilantongo Mixtec +xte I L Ketengban +xtg I A Transalpine Gaulish +xth I E Yitha Yitha +xti I L Sinicahua Mixtec +xtj I L San Juan Teita Mixtec +xtl I L Tijaltepec Mixtec +xtm I L Magdalena Peñasco Mixtec +xtn I L Northern Tlaxiaco Mixtec +xto I A Tokharian A +xtp I L San Miguel Piedras Mixtec +xtq I H Tumshuqese +xtr I A Early Tripuri +xts I L Sindihui Mixtec +xtt I L Tacahua Mixtec +xtu I L Cuyamecalco Mixtec +xtv I E Thawa +xtw I L Tawandê +xty I L Yoloxochitl Mixtec +xtz I E Tasmanian +xua I L Alu Kurumba +xub I L Betta Kurumba +xud I E Umiida +xug I L Kunigami +xuj I L Jennu Kurumba +xul I E Ngunawal +xum I A Umbrian +xun I E Unggaranggu +xuo I L Kuo +xup I E Upper Umpqua +xur I A Urartian +xut I E Kuthant +xuu I L Kxoe +xve I A Venetic +xvi I L Kamviri +xvn I A Vandalic +xvo I A Volscian +xvs I A Vestinian +xwa I L Kwaza +xwc I E Woccon +xwd I E Wadi Wadi +xwe I L Xwela Gbe +xwg I L Kwegu +xwj I E Wajuk +xwk I E Wangkumara +xwl I L Western Xwla Gbe +xwo I E Written Oirat +xwr I L Kwerba Mamberamo +xwt I E Wotjobaluk +xww I E Wemba Wemba +xxb I E Boro (Ghana) +xxk I L Ke'o +xxm I E Minkin +xxr I E Koropó +xxt I E Tambora +xya I E Yaygir +xyb I E Yandjibara +xyj I E Mayi-Yapi +xyk I E Mayi-Kulan +xyl I E Yalakalore +xyt I E Mayi-Thakurti +xyy I L Yorta Yorta +xzh I A Zhang-Zhung +xzm I E Zemgalian +xzp I H Ancient Zapotec +yaa I L Yaminahua +yab I L Yuhup +yac I L Pass Valley Yali +yad I L Yagua +yae I L Pumé +yaf I L Yaka (Democratic Republic of Congo) +yag I L Yámana +yah I L Yazgulyam +yai I L Yagnobi +yaj I L Banda-Yangere +yak I L Yakama +yal I L Yalunka +yam I L Yamba +yan I L Mayangna +yao yao yao I L Yao +yap yap yap I L Yapese +yaq I L Yaqui +yar I L Yabarana +yas I L Nugunu (Cameroon) +yat I L Yambeta +yau I L Yuwana +yav I L Yangben +yaw I L Yawalapití +yax I L Yauma +yay I L Agwagwune +yaz I L Lokaa +yba I L Yala +ybb I L Yemba +ybe I L West Yugur +ybh I L Yakha +ybi I L Yamphu +ybj I L Hasha +ybk I L Bokha +ybl I L Yukuben +ybm I L Yaben +ybn I E Yabaâna +ybo I L Yabong +ybx I L Yawiyo +yby I L Yaweyuha +ych I L Chesu +ycl I L Lolopo +ycn I L Yucuna +ycp I L Chepya +yda I E Yanda +ydd I L Eastern Yiddish +yde I L Yangum Dey +ydg I L Yidgha +ydk I L Yoidik +yds I L Yiddish Sign Language +yea I L Ravula +yec I L Yeniche +yee I L Yimas +yei I E Yeni +yej I L Yevanic +yel I L Yela +yer I L Tarok +yes I L Nyankpa +yet I L Yetfa +yeu I L Yerukula +yev I L Yapunda +yey I L Yeyi +yga I E Malyangapa +ygi I E Yiningayi +ygl I L Yangum Gel +ygm I L Yagomi +ygp I L Gepo +ygr I L Yagaria +ygu I L Yugul +ygw I L Yagwoia +yha I L Baha Buyang +yhd I L Judeo-Iraqi Arabic +yhl I L Hlepho Phowa +yia I L Yinggarda +yid yid yid yi M L Yiddish +yif I L Ache +yig I L Wusa Nasu +yih I L Western Yiddish +yii I L Yidiny +yij I L Yindjibarndi +yik I L Dongshanba Lalo +yil I E Yindjilandji +yim I L Yimchungru Naga +yin I L Yinchia +yip I L Pholo +yiq I L Miqie +yir I L North Awyu +yis I L Yis +yit I L Eastern Lalu +yiu I L Awu +yiv I L Northern Nisu +yix I L Axi Yi +yiz I L Azhe +yka I L Yakan +ykg I L Northern Yukaghir +yki I L Yoke +ykk I L Yakaikeke +ykl I L Khlula +ykm I L Kap +ykn I L Kua-nsi +yko I L Yasa +ykr I L Yekora +ykt I L Kathu +yku I L Kuamasi +yky I L Yakoma +yla I L Yaul +ylb I L Yaleba +yle I L Yele +ylg I L Yelogu +yli I L Angguruk Yali +yll I L Yil +ylm I L Limi +yln I L Langnian Buyang +ylo I L Naluo Yi +ylr I E Yalarnnga +ylu I L Aribwaung +yly I L Nyâlayu +ymb I L Yambes +ymc I L Southern Muji +ymd I L Muda +yme I E Yameo +ymg I L Yamongeri +ymh I L Mili +ymi I L Moji +ymk I L Makwe +yml I L Iamalele +ymm I L Maay +ymn I L Yamna +ymo I L Yangum Mon +ymp I L Yamap +ymq I L Qila Muji +ymr I L Malasar +yms I A Mysian +ymt I E Mator-Taygi-Karagas +ymx I L Northern Muji +ymz I L Muzi +yna I L Aluo +ynd I E Yandruwandha +yne I L Lang'e +yng I L Yango +ynh I L Yangho +ynk I L Naukan Yupik +ynl I L Yangulam +ynn I E Yana +yno I L Yong +ynq I L Yendang +yns I L Yansi +ynu I E Yahuna +yob I E Yoba +yog I L Yogad +yoi I L Yonaguni +yok I L Yokuts +yol I E Yola +yom I L Yombe +yon I L Yongkom +yor yor yor yo I L Yoruba +yot I L Yotti +yox I L Yoron +yoy I L Yoy +ypa I L Phala +ypb I L Labo Phowa +ypg I L Phola +yph I L Phupha +ypm I L Phuma +ypn I L Ani Phowa +ypo I L Alo Phola +ypp I L Phupa +ypz I L Phuza +yra I L Yerakai +yrb I L Yareba +yre I L Yaouré +yri I L Yarí +yrk I L Nenets +yrl I L Nhengatu +yrm I L Yirrk-Mel +yrn I L Yerong +yrs I L Yarsun +yrw I L Yarawata +yry I L Yarluyandi +ysc I E Yassic +ysd I L Samatao +ysg I L Sonaga +ysl I L Yugoslavian Sign Language +ysn I L Sani +yso I L Nisi (China) +ysp I L Southern Lolopo +ysr I E Sirenik Yupik +yss I L Yessan-Mayo +ysy I L Sanie +yta I L Talu +ytl I L Tanglang +ytp I L Thopho +ytw I L Yout Wam +yty I E Yatay +yua I L Yucateco +yub I E Yugambal +yuc I L Yuchi +yud I L Judeo-Tripolitanian Arabic +yue I L Yue Chinese +yuf I L Havasupai-Walapai-Yavapai +yug I E Yug +yui I L Yurutí +yuj I L Karkar-Yuri +yuk I E Yuki +yul I L Yulu +yum I L Quechan +yun I L Bena (Nigeria) +yup I L Yukpa +yuq I L Yuqui +yur I L Yurok +yut I L Yopno +yuu I L Yugh +yuw I L Yau (Morobe Province) +yux I L Southern Yukaghir +yuy I L East Yugur +yuz I L Yuracare +yva I L Yawa +yvt I E Yavitero +ywa I L Kalou +ywg I L Yinhawangka +ywl I L Western Lalu +ywn I L Yawanawa +ywq I L Wuding-Luquan Yi +ywr I L Yawuru +ywt I L Xishanba Lalo +ywu I L Wumeng Nasu +yww I E Yawarawarga +yxa I E Mayawali +yxg I E Yagara +yxl I E Yardliyawarra +yxm I E Yinwum +yxu I E Yuyu +yxy I E Yabula Yabula +yyr I E Yir Yoront +yyu I L Yau (Sandaun Province) +yyz I L Ayizi +yzg I L E'ma Buyang +yzk I L Zokhuo +zaa I L Sierra de Juárez Zapotec +zab I L San Juan Guelavía Zapotec +zac I L Ocotlán Zapotec +zad I L Cajonos Zapotec +zae I L Yareni Zapotec +zaf I L Ayoquesco Zapotec +zag I L Zaghawa +zah I L Zangwal +zai I L Isthmus Zapotec +zaj I L Zaramo +zak I L Zanaki +zal I L Zauzou +zam I L Miahuatlán Zapotec +zao I L Ozolotepec Zapotec +zap zap zap M L Zapotec +zaq I L Aloápam Zapotec +zar I L Rincón Zapotec +zas I L Santo Domingo Albarradas Zapotec +zat I L Tabaa Zapotec +zau I L Zangskari +zav I L Yatzachi Zapotec +zaw I L Mitla Zapotec +zax I L Xadani Zapotec +zay I L Zayse-Zergulla +zaz I L Zari +zbc I L Central Berawan +zbe I L East Berawan +zbl zbl zbl I C Blissymbols +zbt I L Batui +zbw I L West Berawan +zca I L Coatecas Altas Zapotec +zch I L Central Hongshuihe Zhuang +zdj I L Ngazidja Comorian +zea I L Zeeuws +zeg I L Zenag +zeh I L Eastern Hongshuihe Zhuang +zen zen zen I L Zenaga +zga I L Kinga +zgb I L Guibei Zhuang +zgh I L Standard Moroccan Tamazight +zgm I L Minz Zhuang +zgn I L Guibian Zhuang +zgr I L Magori +zha zha zha za M L Zhuang +zhb I L Zhaba +zhd I L Dai Zhuang +zhi I L Zhire +zhn I L Nong Zhuang +zho chi zho zh M L Chinese +zhw I L Zhoa +zia I L Zia +zib I L Zimbabwe Sign Language +zik I L Zimakani +zil I L Zialo +zim I L Mesme +zin I L Zinza +zir I E Ziriya +ziw I L Zigula +ziz I L Zizilivakan +zka I L Kaimbulawa +zkb I E Koibal +zkd I L Kadu +zkg I E Koguryo +zkh I E Khorezmian +zkk I E Karankawa +zkn I L Kanan +zko I E Kott +zkp I E São Paulo Kaingáng +zkr I L Zakhring +zkt I E Kitan +zku I E Kaurna +zkv I E Krevinian +zkz I E Khazar +zlj I L Liujiang Zhuang +zlm I L Malay (individual language) +zln I L Lianshan Zhuang +zlq I L Liuqian Zhuang +zma I L Manda (Australia) +zmb I L Zimba +zmc I E Margany +zmd I L Maridan +zme I E Mangerr +zmf I L Mfinu +zmg I L Marti Ke +zmh I E Makolkol +zmi I L Negeri Sembilan Malay +zmj I L Maridjabin +zmk I E Mandandanyi +zml I L Madngele +zmm I L Marimanindji +zmn I L Mbangwe +zmo I L Molo +zmp I L Mpuono +zmq I L Mituku +zmr I L Maranunggu +zms I L Mbesa +zmt I L Maringarr +zmu I E Muruwari +zmv I E Mbariman-Gudhinma +zmw I L Mbo (Democratic Republic of Congo) +zmx I L Bomitaba +zmy I L Mariyedi +zmz I L Mbandja +zna I L Zan Gula +zne I L Zande (individual language) +zng I L Mang +znk I E Manangkari +zns I L Mangas +zoc I L Copainalá Zoque +zoh I L Chimalapa Zoque +zom I L Zou +zoo I L Asunción Mixtepec Zapotec +zoq I L Tabasco Zoque +zor I L Rayón Zoque +zos I L Francisco León Zoque +zpa I L Lachiguiri Zapotec +zpb I L Yautepec Zapotec +zpc I L Choapan Zapotec +zpd I L Southeastern Ixtlán Zapotec +zpe I L Petapa Zapotec +zpf I L San Pedro Quiatoni Zapotec +zpg I L Guevea De Humboldt Zapotec +zph I L Totomachapan Zapotec +zpi I L Santa María Quiegolani Zapotec +zpj I L Quiavicuzas Zapotec +zpk I L Tlacolulita Zapotec +zpl I L Lachixío Zapotec +zpm I L Mixtepec Zapotec +zpn I L Santa Inés Yatzechi Zapotec +zpo I L Amatlán Zapotec +zpp I L El Alto Zapotec +zpq I L Zoogocho Zapotec +zpr I L Santiago Xanica Zapotec +zps I L Coatlán Zapotec +zpt I L San Vicente Coatlán Zapotec +zpu I L Yalálag Zapotec +zpv I L Chichicapan Zapotec +zpw I L Zaniza Zapotec +zpx I L San Baltazar Loxicha Zapotec +zpy I L Mazaltepec Zapotec +zpz I L Texmelucan Zapotec +zqe I L Qiubei Zhuang +zra I E Kara (Korea) +zrg I L Mirgan +zrn I L Zerenkel +zro I L Záparo +zrp I E Zarphatic +zrs I L Mairasi +zsa I L Sarasira +zsk I A Kaskean +zsl I L Zambian Sign Language +zsm I L Standard Malay +zsr I L Southern Rincon Zapotec +zsu I L Sukurum +zte I L Elotepec Zapotec +ztg I L Xanaguía Zapotec +ztl I L Lapaguía-Guivini Zapotec +ztm I L San Agustín Mixtepec Zapotec +ztn I L Santa Catarina Albarradas Zapotec +ztp I L Loxicha Zapotec +ztq I L Quioquitani-Quierí Zapotec +zts I L Tilquiapan Zapotec +ztt I L Tejalapan Zapotec +ztu I L Güilá Zapotec +ztx I L Zaachila Zapotec +zty I L Yatee Zapotec +zua I L Zeem +zuh I L Tokano +zul zul zul zu I L Zulu +zum I L Kumzari +zun zun zun I L Zuni +zuy I L Zumaya +zwa I L Zay +zxx zxx zxx S S No linguistic content +zyb I L Yongbei Zhuang +zyg I L Yang Zhuang +zyj I L Youjiang Zhuang +zyn I L Yongnan Zhuang +zyp I L Zyphe Chin +zza zza zza M L Zaza zzj I L Zuojiang Zhuang \ No newline at end of file diff --git a/Contents/Libraries/Shared/babelfish/language.py b/Contents/Libraries/Shared/babelfish/language.py index b4b251937..6c98fb602 100644 --- a/Contents/Libraries/Shared/babelfish/language.py +++ b/Contents/Libraries/Shared/babelfish/language.py @@ -4,7 +4,6 @@ # Use of this source code is governed by the 3-clause BSD license # that can be found in the LICENSE file. # -from __future__ import unicode_literals from collections import namedtuple from functools import partial from pkg_resources import resource_stream # @UnresolvedImport diff --git a/Contents/Libraries/Shared/babelfish/script.py b/Contents/Libraries/Shared/babelfish/script.py index 4b59ce016..52258c95a 100644 --- a/Contents/Libraries/Shared/babelfish/script.py +++ b/Contents/Libraries/Shared/babelfish/script.py @@ -4,7 +4,6 @@ # Use of this source code is governed by the 3-clause BSD license # that can be found in the LICENSE file. # -from __future__ import unicode_literals from collections import namedtuple from pkg_resources import resource_stream # @UnresolvedImport from . import basestr diff --git a/Contents/Libraries/Shared/babelfish/tests.py b/Contents/Libraries/Shared/babelfish/tests.py index cf688af92..b72ec284c 100644 --- a/Contents/Libraries/Shared/babelfish/tests.py +++ b/Contents/Libraries/Shared/babelfish/tests.py @@ -212,7 +212,7 @@ def test_converter_opensubtitles(self): self.assertEqual(Language.fromcode('pob', 'opensubtitles'), Language('por', 'BR')) self.assertRaises(LanguageReverseError, lambda: Language.fromopensubtitles('zzz')) self.assertRaises(LanguageConvertError, lambda: Language('aaa').opensubtitles) - self.assertEqual(len(language_converters['opensubtitles'].codes), 606) + self.assertEqual(len(language_converters['opensubtitles'].codes), 607) # test with all the LANGUAGES from the opensubtitles api # downloaded from: http://www.opensubtitles.org/addons/export_languages.php @@ -228,6 +228,10 @@ def test_converter_opensubtitles(self): self.assertEqual(Language.fromopensubtitles(idlang), Language.fromopensubtitles(alpha2)) f.close() + def test_converter_opensubtitles_codes(self): + for code in language_converters['opensubtitles'].from_opensubtitles.keys(): + self.assertIn(code, language_converters['opensubtitles'].codes) + def test_fromietf_country_script(self): language = Language.fromietf('fra-FR-Latn') self.assertEqual(language.alpha3, 'fra') @@ -283,6 +287,11 @@ def test_language_hasattr(self): self.assertTrue(hasattr(Language('fra'), 'alpha2')) self.assertFalse(hasattr(Language('bej'), 'alpha2')) + def test_country_hasattr(self): + self.assertTrue(hasattr(Country('US'), 'name')) + self.assertTrue(hasattr(Country('FR'), 'alpha2')) + self.assertFalse(hasattr(Country('BE'), 'none')) + def test_country(self): self.assertEqual(Language('por', 'BR').country, Country('BR')) self.assertEqual(Language('eng', Country('US')).country, Country('US')) diff --git a/Contents/Libraries/Shared/backports/__init__.py b/Contents/Libraries/Shared/backports/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/Contents/Libraries/Shared/backports/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/Contents/Libraries/Shared/backports/functools_lru_cache.py b/Contents/Libraries/Shared/backports/functools_lru_cache.py new file mode 100644 index 000000000..e0b19d951 --- /dev/null +++ b/Contents/Libraries/Shared/backports/functools_lru_cache.py @@ -0,0 +1,196 @@ +from __future__ import absolute_import + +import functools +from collections import namedtuple +from threading import RLock + +_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) + + +@functools.wraps(functools.update_wrapper) +def update_wrapper( + wrapper, + wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES, +): + """ + Patch two bugs in functools.update_wrapper. + """ + # workaround for http://bugs.python.org/issue3445 + assigned = tuple(attr for attr in assigned if hasattr(wrapped, attr)) + wrapper = functools.update_wrapper(wrapper, wrapped, assigned, updated) + # workaround for https://bugs.python.org/issue17482 + wrapper.__wrapped__ = wrapped + return wrapper + + +class _HashedSeq(list): + __slots__ = 'hashvalue' + + def __init__(self, tup, hash=hash): + self[:] = tup + self.hashvalue = hash(tup) + + def __hash__(self): + return self.hashvalue + + +def _make_key( + args, + kwds, + typed, + kwd_mark=(object(),), + fasttypes=set([int, str, frozenset, type(None)]), + sorted=sorted, + tuple=tuple, + type=type, + len=len, +): + 'Make a cache key from optionally typed positional and keyword arguments' + key = args + if kwds: + sorted_items = sorted(kwds.items()) + key += kwd_mark + for item in sorted_items: + key += item + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for k, v in sorted_items) + elif len(key) == 1 and type(key[0]) in fasttypes: + return key[0] + return _HashedSeq(key) + + +def lru_cache(maxsize=100, typed=False): + """Least-recently-used cache decorator. + + If *maxsize* is set to None, the LRU features are disabled and the cache + can grow without bound. + + If *typed* is True, arguments of different types will be cached separately. + For example, f(3.0) and f(3) will be treated as distinct calls with + distinct results. + + Arguments to the cached function must be hashable. + + View the cache statistics named tuple (hits, misses, maxsize, currsize) with + f.cache_info(). Clear the cache and statistics with f.cache_clear(). + Access the underlying function with f.__wrapped__. + + See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used + + """ + + # Users should only access the lru_cache through its public API: + # cache_info, cache_clear, and f.__wrapped__ + # The internals of the lru_cache are encapsulated for thread safety and + # to allow the implementation to change (including a possible C version). + + def decorating_function(user_function): + + cache = dict() + stats = [0, 0] # make statistics updateable non-locally + HITS, MISSES = 0, 1 # names for the stats fields + make_key = _make_key + cache_get = cache.get # bound method to lookup key or return None + _len = len # localize the global len() function + lock = RLock() # because linkedlist updates aren't threadsafe + root = [] # root of the circular doubly linked list + root[:] = [root, root, None, None] # initialize by pointing to self + nonlocal_root = [root] # make updateable non-locally + PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + + if maxsize == 0: + + def wrapper(*args, **kwds): + # no caching, just do a statistics update after a successful call + result = user_function(*args, **kwds) + stats[MISSES] += 1 + return result + + elif maxsize is None: + + def wrapper(*args, **kwds): + # simple caching without ordering or size limit + key = make_key(args, kwds, typed) + result = cache_get( + key, root + ) # root used here as a unique not-found sentinel + if result is not root: + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + cache[key] = result + stats[MISSES] += 1 + return result + + else: + + def wrapper(*args, **kwds): + # size limited caching that tracks accesses by recency + key = make_key(args, kwds, typed) if kwds or typed else args + with lock: + link = cache_get(key) + if link is not None: + # record recent use of the key by moving it + # to the front of the list + root, = nonlocal_root + link_prev, link_next, key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = root[PREV] + last[NEXT] = root[PREV] = link + link[PREV] = last + link[NEXT] = root + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + with lock: + root, = nonlocal_root + if key in cache: + # getting here means that this same key was added to the + # cache while the lock was released. since the link + # update is already done, we need only return the + # computed result and update the count of misses. + pass + elif _len(cache) >= maxsize: + # use the old root to store the new key and result + oldroot = root + oldroot[KEY] = key + oldroot[RESULT] = result + # empty the oldest link and make it the new root + root = nonlocal_root[0] = oldroot[NEXT] + oldkey = root[KEY] + root[KEY] = root[RESULT] = None + # now update the cache dictionary for the new links + del cache[oldkey] + cache[key] = oldroot + else: + # put result in a new link at the front of the list + last = root[PREV] + link = [last, root, key, result] + last[NEXT] = root[PREV] = cache[key] = link + stats[MISSES] += 1 + return result + + def cache_info(): + """Report cache statistics""" + with lock: + return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache)) + + def cache_clear(): + """Clear the cache and cache statistics""" + with lock: + cache.clear() + root = nonlocal_root[0] + root[:] = [root, root, None, None] + stats[:] = [0, 0] + + wrapper.__wrapped__ = user_function + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return update_wrapper(wrapper, user_function) + + return decorating_function diff --git a/Contents/Libraries/Shared/bs4/COPYING.txt b/Contents/Libraries/Shared/bs4/COPYING.txt index d668d13f0..b91188869 100644 --- a/Contents/Libraries/Shared/bs4/COPYING.txt +++ b/Contents/Libraries/Shared/bs4/COPYING.txt @@ -1,6 +1,6 @@ Beautiful Soup is made available under the MIT license: - Copyright (c) 2004-2012 Leonard Richardson + Copyright (c) 2004-2015 Leonard Richardson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -20,7 +20,8 @@ Beautiful Soup is made available under the MIT license: BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE, DAMMIT. + SOFTWARE. Beautiful Soup incorporates code from the html5lib library, which is -also made available under the MIT license. +also made available under the MIT license. Copyright (c) 2006-2013 +James Graham and other contributors diff --git a/Contents/Libraries/Shared/bs4/NEWS.txt b/Contents/Libraries/Shared/bs4/NEWS.txt index 88a60a245..3726c570a 100644 --- a/Contents/Libraries/Shared/bs4/NEWS.txt +++ b/Contents/Libraries/Shared/bs4/NEWS.txt @@ -1,3 +1,127 @@ += 4.4.1 (20150928) = + +* Fixed a bug that deranged the tree when part of it was + removed. Thanks to Eric Weiser for the patch and John Wiseman for a + test. [bug=1481520] + +* Fixed a parse bug with the html5lib tree-builder. Thanks to Roel + Kramer for the patch. [bug=1483781] + +* Improved the implementation of CSS selector grouping. Thanks to + Orangain for the patch. [bug=1484543] + +* Fixed the test_detect_utf8 test so that it works when chardet is + installed. [bug=1471359] + +* Corrected the output of Declaration objects. [bug=1477847] + + += 4.4.0 (20150703) = + +Especially important changes: + +* Added a warning when you instantiate a BeautifulSoup object without + explicitly naming a parser. [bug=1398866] + +* __repr__ now returns an ASCII bytestring in Python 2, and a Unicode + string in Python 3, instead of a UTF8-encoded bytestring in both + versions. In Python 3, __str__ now returns a Unicode string instead + of a bytestring. [bug=1420131] + +* The `text` argument to the find_* methods is now called `string`, + which is more accurate. `text` still works, but `string` is the + argument described in the documentation. `text` may eventually + change its meaning, but not for a very long time. [bug=1366856] + +* Changed the way soup objects work under copy.copy(). Copying a + NavigableString or a Tag will give you a new NavigableString that's + equal to the old one but not connected to the parse tree. Patch by + Martijn Peters. [bug=1307490] + +* Started using a standard MIT license. [bug=1294662] + +* Added a Chinese translation of the documentation by Delong .w. + +New features: + +* Introduced the select_one() method, which uses a CSS selector but + only returns the first match, instead of a list of + matches. [bug=1349367] + +* You can now create a Tag object without specifying a + TreeBuilder. Patch by Martijn Pieters. [bug=1307471] + +* You can now create a NavigableString or a subclass just by invoking + the constructor. [bug=1294315] + +* Added an `exclude_encodings` argument to UnicodeDammit and to the + Beautiful Soup constructor, which lets you prohibit the detection of + an encoding that you know is wrong. [bug=1469408] + +* The select() method now supports selector grouping. Patch by + Francisco Canas [bug=1191917] + +Bug fixes: + +* Fixed yet another problem that caused the html5lib tree builder to + create a disconnected parse tree. [bug=1237763] + +* Force object_was_parsed() to keep the tree intact even when an element + from later in the document is moved into place. [bug=1430633] + +* Fixed yet another bug that caused a disconnected tree when html5lib + copied an element from one part of the tree to another. [bug=1270611] + +* Fixed a bug where Element.extract() could create an infinite loop in + the remaining tree. + +* The select() method can now find tags whose names contain + dashes. Patch by Francisco Canas. [bug=1276211] + +* The select() method can now find tags with attributes whose names + contain dashes. Patch by Marek Kapolka. [bug=1304007] + +* Improved the lxml tree builder's handling of processing + instructions. [bug=1294645] + +* Restored the helpful syntax error that happens when you try to + import the Python 2 edition of Beautiful Soup under Python + 3. [bug=1213387] + +* In Python 3.4 and above, set the new convert_charrefs argument to + the html.parser constructor to avoid a warning and future + failures. Patch by Stefano Revera. [bug=1375721] + +* The warning when you pass in a filename or URL as markup will now be + displayed correctly even if the filename or URL is a Unicode + string. [bug=1268888] + +* If the initial <html> tag contains a CDATA list attribute such as + 'class', the html5lib tree builder will now turn its value into a + list, as it would with any other tag. [bug=1296481] + +* Fixed an import error in Python 3.5 caused by the removal of the + HTMLParseError class. [bug=1420063] + +* Improved docstring for encode_contents() and + decode_contents(). [bug=1441543] + +* Fixed a crash in Unicode, Dammit's encoding detector when the name + of the encoding itself contained invalid bytes. [bug=1360913] + +* Improved the exception raised when you call .unwrap() or + .replace_with() on an element that's not attached to a tree. + +* Raise a NotImplementedError whenever an unsupported CSS pseudoclass + is used in select(). Previously some cases did not result in a + NotImplementedError. + +* It's now possible to pickle a BeautifulSoup object no matter which + tree builder was used to create it. However, the only tree builder + that survives the pickling process is the HTMLParserTreeBuilder + ('html.parser'). If you unpickle a BeautifulSoup object created with + some other tree builder, soup.builder will be None. [bug=1231545] + = 4.3.2 (20131002) = * Fixed a bug in which short Unicode input was improperly encoded to diff --git a/Contents/Libraries/Shared/bs4/TODO.txt b/Contents/Libraries/Shared/bs4/TODO.txt new file mode 100644 index 000000000..e26d6264d --- /dev/null +++ b/Contents/Libraries/Shared/bs4/TODO.txt @@ -0,0 +1,31 @@ +Additions +--------- + +More of the jQuery API: nextUntil? + +Optimizations +------------- + +The html5lib tree builder doesn't use the standard tree-building API, +which worries me and has resulted in a number of bugs. + +markup_attr_map can be optimized since it's always a map now. + +Upon encountering UTF-16LE data or some other uncommon serialization +of Unicode, UnicodeDammit will convert the data to Unicode, then +encode it at UTF-8. This is wasteful because it will just get decoded +back to Unicode. + +CDATA +----- + +The elementtree XMLParser has a strip_cdata argument that, when set to +False, should allow Beautiful Soup to preserve CDATA sections instead +of treating them as text. Except it doesn't. (This argument is also +present for HTMLParser, and also does nothing there.) + +Currently, htm5lib converts CDATA sections into comments. An +as-yet-unreleased version of html5lib changes the parser's handling of +CDATA sections to allow CDATA sections in tags like <svg> and +<math>. The HTML5TreeBuilder will need to be updated to create CData +objects instead of Comment objects in this situation. diff --git a/Contents/Libraries/Shared/bs4/__init__.py b/Contents/Libraries/Shared/bs4/__init__.py index 7ba34269a..7a80452f7 100644 --- a/Contents/Libraries/Shared/bs4/__init__.py +++ b/Contents/Libraries/Shared/bs4/__init__.py @@ -5,26 +5,31 @@ Beautiful Soup uses a pluggable XML or HTML parser to parse a (possibly invalid) document into a tree representation. Beautiful Soup -provides provides methods and Pythonic idioms that make it easy to -navigate, search, and modify the parse tree. +provides methods and Pythonic idioms that make it easy to navigate, +search, and modify the parse tree. -Beautiful Soup works with Python 2.6 and up. It works better if lxml +Beautiful Soup works with Python 2.7 and up. It works better if lxml and/or html5lib is installed. For more than you ever wanted to know about Beautiful Soup, see the documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/ + """ +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + __author__ = "Leonard Richardson (leonardr@segfault.org)" -__version__ = "4.3.2" -__copyright__ = "Copyright (c) 2004-2013 Leonard Richardson" +__version__ = "4.6.0" +__copyright__ = "Copyright (c) 2004-2017 Leonard Richardson" __license__ = "MIT" __all__ = ['BeautifulSoup'] import os import re +import traceback import warnings from .builder import builder_registry, ParserRejectedMarkup @@ -45,7 +50,7 @@ # The very first thing we do is give a useful error if someone is # running this code under Python 3 without converting it. -syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).' +'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'<>'You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).' class BeautifulSoup(Tag): """ @@ -77,8 +82,11 @@ class BeautifulSoup(Tag): ASCII_SPACES = '\x20\x0a\x09\x0c\x0d' + NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nThe code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, change code that looks like this:\n\n BeautifulSoup(YOUR_MARKUP})\n\nto this:\n\n BeautifulSoup(YOUR_MARKUP, \"%(parser)s\")\n" + def __init__(self, markup="", features=None, builder=None, - parse_only=None, from_encoding=None, **kwargs): + parse_only=None, from_encoding=None, exclude_encodings=None, + **kwargs): """The Soup object is initialized as the 'root tag', and the provided markup (which can be a string or a file-like object) is fed into the underlying parser.""" @@ -114,9 +122,9 @@ def __init__(self, markup="", features=None, builder=None, del kwargs['isHTML'] warnings.warn( "BS4 does not respect the isHTML argument to the " - "BeautifulSoup constructor. You can pass in features='html' " - "or features='xml' to get a builder capable of handling " - "one or the other.") + "BeautifulSoup constructor. Suggest you use " + "features='lxml' for HTML and features='lxml-xml' for " + "XML.") def deprecated_argument(old_name, new_name): if old_name in kwargs: @@ -134,12 +142,17 @@ def deprecated_argument(old_name, new_name): from_encoding = from_encoding or deprecated_argument( "fromEncoding", "from_encoding") + if from_encoding and isinstance(markup, unicode): + warnings.warn("You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.") + from_encoding = None + if len(kwargs) > 0: arg = kwargs.keys().pop() raise TypeError( "__init__() got an unexpected keyword argument '%s'" % arg) if builder is None: + original_features = features if isinstance(features, basestring): features = [features] if features is None or len(features) == 0: @@ -151,15 +164,35 @@ def deprecated_argument(old_name, new_name): "requested: %s. Do you need to install a parser library?" % ",".join(features)) builder = builder_class() + if not (original_features == builder.NAME or + original_features in builder.ALTERNATE_NAMES): + if builder.is_xml: + markup_type = "XML" + else: + markup_type = "HTML" + + caller = traceback.extract_stack()[0] + filename = caller[0] + line_number = caller[1] + warnings.warn(self.NO_PARSER_SPECIFIED_WARNING % dict( + filename=filename, + line_number=line_number, + parser=builder.NAME, + markup_type=markup_type)) + self.builder = builder self.is_xml = builder.is_xml + self.known_xml = self.is_xml self.builder.soup = self self.parse_only = parse_only if hasattr(markup, 'read'): # It's a file-type object. markup = markup.read() - elif len(markup) <= 256: + elif len(markup) <= 256 and ( + (isinstance(markup, bytes) and not b'<' in markup) + or (isinstance(markup, unicode) and not u'<' in markup) + ): # Print out warnings for a couple beginner problems # involving passing non-markup to Beautiful Soup. # Beautiful Soup will still parse the input as markup, @@ -178,19 +211,18 @@ def deprecated_argument(old_name, new_name): # system. Just let it go. pass if is_file: + if isinstance(markup, unicode): + markup = markup.encode("utf8") warnings.warn( - '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup) - if markup[:5] == "http:" or markup[:6] == "https:": - # TODO: This is ugly but I couldn't get it to work in - # Python 3 otherwise. - if ((isinstance(markup, bytes) and not b' ' in markup) - or (isinstance(markup, unicode) and not u' ' in markup)): - warnings.warn( - '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup) + '"%s" looks like a filename, not markup. You should' + ' probably open this file and pass the filehandle into' + ' Beautiful Soup.' % markup) + self._check_markup_is_url(markup) for (self.markup, self.original_encoding, self.declared_html_encoding, self.contains_replacement_characters) in ( - self.builder.prepare_markup(markup, from_encoding)): + self.builder.prepare_markup( + markup, from_encoding, exclude_encodings=exclude_encodings)): self.reset() try: self._feed() @@ -203,6 +235,53 @@ def deprecated_argument(old_name, new_name): self.markup = None self.builder.soup = None + def __copy__(self): + copy = type(self)( + self.encode('utf-8'), builder=self.builder, from_encoding='utf-8' + ) + + # Although we encoded the tree to UTF-8, that may not have + # been the encoding of the original markup. Set the copy's + # .original_encoding to reflect the original object's + # .original_encoding. + copy.original_encoding = self.original_encoding + return copy + + def __getstate__(self): + # Frequently a tree builder can't be pickled. + d = dict(self.__dict__) + if 'builder' in d and not self.builder.picklable: + d['builder'] = None + return d + + @staticmethod + def _check_markup_is_url(markup): + """ + Check if markup looks like it's actually a url and raise a warning + if so. Markup can be unicode or str (py2) / bytes (py3). + """ + if isinstance(markup, bytes): + space = b' ' + cant_start_with = (b"http:", b"https:") + elif isinstance(markup, unicode): + space = u' ' + cant_start_with = (u"http:", u"https:") + else: + return + + if any(markup.startswith(prefix) for prefix in cant_start_with): + if not space in markup: + if isinstance(markup, bytes): + decoded_markup = markup.decode('utf-8', 'replace') + else: + decoded_markup = markup + warnings.warn( + '"%s" looks like a URL. Beautiful Soup is not an' + ' HTTP client. You should probably use an HTTP client like' + ' requests to get the document behind the URL, and feed' + ' that document to Beautiful Soup.' % decoded_markup + ) + def _feed(self): # Convert the document to Unicode. self.builder.reset() @@ -229,9 +308,7 @@ def new_tag(self, name, namespace=None, nsprefix=None, **attrs): def new_string(self, s, subclass=NavigableString): """Create a new NavigableString associated with this soup.""" - navigable = subclass(s) - navigable.setup() - return navigable + return subclass(s) def insert_before(self, successor): raise NotImplementedError("BeautifulSoup objects don't support insert_before().") @@ -290,14 +367,60 @@ def endData(self, containerClass=NavigableString): def object_was_parsed(self, o, parent=None, most_recent_element=None): """Add an object to the parse tree.""" parent = parent or self.currentTag - most_recent_element = most_recent_element or self._most_recent_element - o.setup(parent, most_recent_element) + previous_element = most_recent_element or self._most_recent_element + + next_element = previous_sibling = next_sibling = None + if isinstance(o, Tag): + next_element = o.next_element + next_sibling = o.next_sibling + previous_sibling = o.previous_sibling + if not previous_element: + previous_element = o.previous_element + + o.setup(parent, previous_element, next_element, previous_sibling, next_sibling) - if most_recent_element is not None: - most_recent_element.next_element = o self._most_recent_element = o parent.contents.append(o) + if parent.next_sibling: + # This node is being inserted into an element that has + # already been parsed. Deal with any dangling references. + index = len(parent.contents)-1 + while index >= 0: + if parent.contents[index] is o: + break + index -= 1 + else: + raise ValueError( + "Error building tree: supposedly %r was inserted " + "into %r after the fact, but I don't see it!" % ( + o, parent + ) + ) + if index == 0: + previous_element = parent + previous_sibling = None + else: + previous_element = previous_sibling = parent.contents[index-1] + if index == len(parent.contents)-1: + next_element = parent.next_sibling + next_sibling = None + else: + next_element = next_sibling = parent.contents[index+1] + + o.previous_element = previous_element + if previous_element: + previous_element.next_element = o + o.next_element = next_element + if next_element: + next_element.previous_element = o + o.next_sibling = next_sibling + if next_sibling: + next_sibling.previous_sibling = o + o.previous_sibling = previous_sibling + if previous_sibling: + previous_sibling.next_sibling = o + def _popToTag(self, name, nsprefix=None, inclusivePop=True): """Pops the tag stack up to and including the most recent instance of the given tag. If inclusivePop is false, pops the tag @@ -325,7 +448,7 @@ def handle_starttag(self, name, namespace, nsprefix, attrs): """Push a start tag on to the stack. If this method returns None, the tag was rejected by the - SoupStrainer. You should proceed as if the tag had not occured + SoupStrainer. You should proceed as if the tag had not occurred in the document. For instance, if this was a self-closing tag, don't call handle_endtag. """ diff --git a/Contents/Libraries/Shared/bs4/builder/__init__.py b/Contents/Libraries/Shared/bs4/builder/__init__.py index 740f5f29c..fdb3362fc 100644 --- a/Contents/Libraries/Shared/bs4/builder/__init__.py +++ b/Contents/Libraries/Shared/bs4/builder/__init__.py @@ -1,9 +1,13 @@ +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + from collections import defaultdict import itertools import sys from bs4.element import ( CharsetMetaAttributeValue, ContentMetaAttributeValue, + HTMLAwareEntitySubstitution, whitespace_re ) @@ -80,9 +84,12 @@ def lookup(self, *features): class TreeBuilder(object): """Turn a document into a Beautiful Soup object tree.""" + NAME = "[Unknown tree builder]" + ALTERNATE_NAMES = [] features = [] is_xml = False + picklable = False preserve_whitespace_tags = set() empty_element_tags = None # A tag will be considered an empty-element # tag when and only when it has no contents. @@ -224,9 +231,14 @@ class HTMLTreeBuilder(TreeBuilder): Such as which tags are empty-element tags. """ - preserve_whitespace_tags = set(['pre', 'textarea']) - empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta', - 'spacer', 'link', 'frame', 'base']) + preserve_whitespace_tags = HTMLAwareEntitySubstitution.preserve_whitespace_tags + empty_element_tags = set([ + # These are from HTML5. + 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr', + + # These are from HTML4, removed in HTML5. + 'spacer', 'frame' + ]) # The HTML standard defines these attributes as containing a # space-separated list of values, not a single value. That is, diff --git a/Contents/Libraries/Shared/bs4/builder/_html5lib.py b/Contents/Libraries/Shared/bs4/builder/_html5lib.py index 7de36ae75..5f5489358 100644 --- a/Contents/Libraries/Shared/bs4/builder/_html5lib.py +++ b/Contents/Libraries/Shared/bs4/builder/_html5lib.py @@ -1,17 +1,27 @@ +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + __all__ = [ 'HTML5TreeBuilder', ] import warnings +import re from bs4.builder import ( PERMISSIVE, HTML, HTML_5, HTMLTreeBuilder, ) -from bs4.element import NamespacedAttribute +from bs4.element import ( + NamespacedAttribute, + whitespace_re, +) import html5lib -from html5lib.constants import namespaces +from html5lib.constants import ( + namespaces, + prefixes, + ) from bs4.element import ( Comment, Doctype, @@ -19,14 +29,32 @@ Tag, ) +try: + # Pre-0.99999999 + from html5lib.treebuilders import _base as treebuilder_base + new_html5lib = False +except ImportError, e: + # 0.99999999 and up + from html5lib.treebuilders import base as treebuilder_base + new_html5lib = True + class HTML5TreeBuilder(HTMLTreeBuilder): """Use html5lib to build a tree.""" - features = ['html5lib', PERMISSIVE, HTML_5, HTML] + NAME = "html5lib" + + features = [NAME, PERMISSIVE, HTML_5, HTML] - def prepare_markup(self, markup, user_specified_encoding): + def prepare_markup(self, markup, user_specified_encoding, + document_declared_encoding=None, exclude_encodings=None): # Store the user-specified encoding for use later on. self.user_specified_encoding = user_specified_encoding + + # document_declared_encoding and exclude_encodings aren't used + # ATM because the html5lib TreeBuilder doesn't use + # UnicodeDammit. + if exclude_encodings: + warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.") yield (markup, None, None, False) # These methods are defined by Beautiful Soup. @@ -34,7 +62,14 @@ def feed(self, markup): if self.soup.parse_only is not None: warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.") parser = html5lib.HTMLParser(tree=self.create_treebuilder) - doc = parser.parse(markup, encoding=self.user_specified_encoding) + + extra_kwargs = dict() + if not isinstance(markup, unicode): + if new_html5lib: + extra_kwargs['override_encoding'] = self.user_specified_encoding + else: + extra_kwargs['encoding'] = self.user_specified_encoding + doc = parser.parse(markup, **extra_kwargs) # Set the character encoding detected by the tokenizer. if isinstance(markup, unicode): @@ -42,11 +77,17 @@ def feed(self, markup): # charEncoding to UTF-8 if it gets Unicode input. doc.original_encoding = None else: - doc.original_encoding = parser.tokenizer.stream.charEncoding[0] + original_encoding = parser.tokenizer.stream.charEncoding[0] + if not isinstance(original_encoding, basestring): + # In 0.99999999 and up, the encoding is an html5lib + # Encoding object. We want to use a string for compatibility + # with other tree builders. + original_encoding = original_encoding.name + doc.original_encoding = original_encoding def create_treebuilder(self, namespaceHTMLElements): self.underlying_builder = TreeBuilderForHtml5lib( - self.soup, namespaceHTMLElements) + namespaceHTMLElements, self.soup) return self.underlying_builder def test_fragment_to_document(self, fragment): @@ -54,10 +95,14 @@ def test_fragment_to_document(self, fragment): return u'<html><head></head><body>%s</body></html>' % fragment -class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder): +class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder): - def __init__(self, soup, namespaceHTMLElements): - self.soup = soup + def __init__(self, namespaceHTMLElements, soup=None): + if soup: + self.soup = soup + else: + from bs4 import BeautifulSoup + self.soup = BeautifulSoup("", "html.parser") super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements) def documentClass(self): @@ -80,7 +125,8 @@ def commentClass(self, data): return TextNode(Comment(data), self.soup) def fragmentClass(self): - self.soup = BeautifulSoup("") + from bs4 import BeautifulSoup + self.soup = BeautifulSoup("", "html.parser") self.soup.name = "[document_fragment]" return Element(self.soup, self.soup, None) @@ -92,7 +138,57 @@ def getDocument(self): return self.soup def getFragment(self): - return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element + return treebuilder_base.TreeBuilder.getFragment(self).element + + def testSerializer(self, element): + from bs4 import BeautifulSoup + rv = [] + doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$') + + def serializeElement(element, indent=0): + if isinstance(element, BeautifulSoup): + pass + if isinstance(element, Doctype): + m = doctype_re.match(element) + if m: + name = m.group(1) + if m.lastindex > 1: + publicId = m.group(2) or "" + systemId = m.group(3) or m.group(4) or "" + rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" % + (' ' * indent, name, publicId, systemId)) + else: + rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name)) + else: + rv.append("|%s<!DOCTYPE >" % (' ' * indent,)) + elif isinstance(element, Comment): + rv.append("|%s<!-- %s -->" % (' ' * indent, element)) + elif isinstance(element, NavigableString): + rv.append("|%s\"%s\"" % (' ' * indent, element)) + else: + if element.namespace: + name = "%s %s" % (prefixes[element.namespace], + element.name) + else: + name = element.name + rv.append("|%s<%s>" % (' ' * indent, name)) + if element.attrs: + attributes = [] + for name, value in element.attrs.items(): + if isinstance(name, NamespacedAttribute): + name = "%s %s" % (prefixes[name.namespace], name.name) + if isinstance(value, list): + value = " ".join(value) + attributes.append((name, value)) + + for name, value in sorted(attributes): + rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) + indent += 2 + for child in element.children: + serializeElement(child, indent) + serializeElement(element, 0) + + return "\n".join(rv) class AttrList(object): def __init__(self, element): @@ -101,7 +197,16 @@ def __init__(self, element): def __iter__(self): return list(self.attrs.items()).__iter__() def __setitem__(self, name, value): - "set attr", name, value + # If this attribute is a multi-valued attribute for this element, + # turn its value into a list. + list_attr = HTML5TreeBuilder.cdata_list_attributes + if (name in list_attr['*'] + or (self.element.name in list_attr + and name in list_attr[self.element.name])): + # A node that is being cloned may have already undergone + # this procedure. + if not isinstance(value, list): + value = whitespace_re.split(value) self.element[name] = value def items(self): return list(self.attrs.items()) @@ -115,9 +220,9 @@ def __contains__(self, name): return name in list(self.attrs.keys()) -class Element(html5lib.treebuilders._base.Node): +class Element(treebuilder_base.Node): def __init__(self, element, soup, namespace): - html5lib.treebuilders._base.Node.__init__(self, element.name) + treebuilder_base.Node.__init__(self, element.name) self.element = element self.soup = soup self.namespace = namespace @@ -136,8 +241,10 @@ def appendChild(self, node): child = node elif node.element.__class__ == NavigableString: string_child = child = node.element + node.parent = self else: child = node.element + node.parent = self if not isinstance(child, basestring) and child.parent is not None: node.element.extract() @@ -161,6 +268,12 @@ def appendChild(self, node): # immediately after the parent, if it has no children.) if self.element.contents: most_recent_element = self.element._last_descendant(False) + elif self.element.next_element is not None: + # Something from further ahead in the parse tree is + # being inserted into this earlier element. This is + # very annoying because it means an expensive search + # for the last element in the tree. + most_recent_element = self.soup._last_descendant() else: most_recent_element = self.element @@ -169,9 +282,12 @@ def appendChild(self, node): most_recent_element=most_recent_element) def getAttributes(self): + if isinstance(self.element, Comment): + return {} return AttrList(self.element) def setAttributes(self, attributes): + if attributes is not None and len(attributes) > 0: converted_attributes = [] @@ -195,11 +311,11 @@ def setAttributes(self, attributes): attributes = property(getAttributes, setAttributes) def insertText(self, data, insertBefore=None): + text = TextNode(self.soup.new_string(data), self.soup) if insertBefore: - text = TextNode(self.soup.new_string(data), self.soup) - self.insertBefore(data, insertBefore) + self.insertBefore(text, insertBefore) else: - self.appendChild(data) + self.appendChild(text) def insertBefore(self, node, refNode): index = self.element.index(refNode.element) @@ -218,6 +334,10 @@ def removeChild(self, node): def reparentChildren(self, new_parent): """Move all of this tag's children into another tag.""" + # print "MOVE", self.element.contents + # print "FROM", self.element + # print "TO", new_parent.element + element = self.element new_parent_element = new_parent.element # Determine what this tag's next_element will be once all the children @@ -236,18 +356,35 @@ def reparentChildren(self, new_parent): new_parents_last_descendant_next_element = new_parent_element.next_element to_append = element.contents - append_after = new_parent.element.contents if len(to_append) > 0: # Set the first child's previous_element and previous_sibling # to elements within the new parent first_child = to_append[0] - first_child.previous_element = new_parents_last_descendant + if new_parents_last_descendant: + first_child.previous_element = new_parents_last_descendant + else: + first_child.previous_element = new_parent_element first_child.previous_sibling = new_parents_last_child - - # Fix the last child's next_element and next_sibling - last_child = to_append[-1] - last_child.next_element = new_parents_last_descendant_next_element - last_child.next_sibling = None + if new_parents_last_descendant: + new_parents_last_descendant.next_element = first_child + else: + new_parent_element.next_element = first_child + if new_parents_last_child: + new_parents_last_child.next_sibling = first_child + + # Find the very last element being moved. It is now the + # parent's last descendant. It has no .next_sibling and + # its .next_element is whatever the previous last + # descendant had. + last_childs_last_descendant = to_append[-1]._last_descendant(False, True) + + last_childs_last_descendant.next_element = new_parents_last_descendant_next_element + if new_parents_last_descendant_next_element: + # TODO: This code has no test coverage and I'm not sure + # how to get html5lib to go through this path, but it's + # just the other side of the previous line. + new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant + last_childs_last_descendant.next_sibling = None for child in to_append: child.parent = new_parent_element @@ -257,6 +394,10 @@ def reparentChildren(self, new_parent): element.contents = [] element.next_element = final_next_element + # print "DONE WITH MOVE" + # print "FROM", self.element + # print "TO", new_parent_element + def cloneNode(self): tag = self.soup.new_tag(self.element.name, self.namespace) node = Element(tag, self.soup, self.namespace) @@ -277,7 +418,7 @@ def getNameTuple(self): class TextNode(Element): def __init__(self, element, soup): - html5lib.treebuilders._base.Node.__init__(self, None) + treebuilder_base.Node.__init__(self, None) self.element = element self.soup = soup diff --git a/Contents/Libraries/Shared/bs4/builder/_htmlparser.py b/Contents/Libraries/Shared/bs4/builder/_htmlparser.py index ca8d8b892..67890b3a3 100644 --- a/Contents/Libraries/Shared/bs4/builder/_htmlparser.py +++ b/Contents/Libraries/Shared/bs4/builder/_htmlparser.py @@ -1,13 +1,22 @@ """Use the HTMLParser library to parse HTML files that aren't too bad.""" +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + __all__ = [ 'HTMLParserTreeBuilder', ] -from HTMLParser import ( - HTMLParser, - HTMLParseError, - ) +from HTMLParser import HTMLParser + +try: + from HTMLParser import HTMLParseError +except ImportError, e: + # HTMLParseError is removed in Python 3.5. Since it can never be + # thrown in 3.5, we can just define our own class as a placeholder. + class HTMLParseError(Exception): + pass + import sys import warnings @@ -19,10 +28,10 @@ # At the end of this file, we monkeypatch HTMLParser so that # strict=True works well on Python 3.2.2. major, minor, release = sys.version_info[:3] -CONSTRUCTOR_TAKES_STRICT = ( - major > 3 - or (major == 3 and minor > 2) - or (major == 3 and minor == 2 and release >= 3)) +CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3 +CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3 +CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4 + from bs4.element import ( CData, @@ -43,7 +52,31 @@ HTMLPARSER = 'html.parser' class BeautifulSoupHTMLParser(HTMLParser): - def handle_starttag(self, name, attrs): + + def __init__(self, *args, **kwargs): + HTMLParser.__init__(self, *args, **kwargs) + + # Keep a list of empty-element tags that were encountered + # without an explicit closing tag. If we encounter a closing tag + # of this type, we'll associate it with one of those entries. + # + # This isn't a stack because we don't care about the + # order. It's a list of closing tags we've already handled and + # will ignore, assuming they ever show up. + self.already_closed_empty_element = [] + + def handle_startendtag(self, name, attrs): + # This is only called when the markup looks like + # <tag/>. + + # is_startend() tells handle_starttag not to close the tag + # just because its name matches a known empty-element tag. We + # know that this is an empty-element tag and we want to call + # handle_endtag ourselves. + tag = self.handle_starttag(name, attrs, handle_empty_element=False) + self.handle_endtag(name) + + def handle_starttag(self, name, attrs, handle_empty_element=True): # XXX namespace attr_dict = {} for key, value in attrs: @@ -53,17 +86,42 @@ def handle_starttag(self, name, attrs): value = '' attr_dict[key] = value attrvalue = '""' - self.soup.handle_starttag(name, None, None, attr_dict) - - def handle_endtag(self, name): - self.soup.handle_endtag(name) + #print "START", name + tag = self.soup.handle_starttag(name, None, None, attr_dict) + if tag and tag.is_empty_element and handle_empty_element: + # Unlike other parsers, html.parser doesn't send separate end tag + # events for empty-element tags. (It's handled in + # handle_startendtag, but only if the original markup looked like + # <tag/>.) + # + # So we need to call handle_endtag() ourselves. Since we + # know the start event is identical to the end event, we + # don't want handle_endtag() to cross off any previous end + # events for tags of this name. + self.handle_endtag(name, check_already_closed=False) + + # But we might encounter an explicit closing tag for this tag + # later on. If so, we want to ignore it. + self.already_closed_empty_element.append(name) + + def handle_endtag(self, name, check_already_closed=True): + #print "END", name + if check_already_closed and name in self.already_closed_empty_element: + # This is a redundant end tag for an empty-element tag. + # We've already called handle_endtag() for it, so just + # check it off the list. + # print "ALREADY CLOSED", name + self.already_closed_empty_element.remove(name) + else: + self.soup.handle_endtag(name) def handle_data(self, data): self.soup.handle_data(data) def handle_charref(self, name): # XXX workaround for a bug in HTMLParser. Remove this once - # it's fixed. + # it's fixed in all supported versions. + # http://bugs.python.org/issue13633 if name.startswith('x'): real_name = int(name.lstrip('x'), 16) elif name.startswith('X'): @@ -113,14 +171,6 @@ def unknown_decl(self, data): def handle_pi(self, data): self.soup.endData() - if data.endswith("?") and data.lower().startswith("xml"): - # "An XHTML processing instruction using the trailing '?' - # will cause the '?' to be included in data." - HTMLParser - # docs. - # - # Strip the question mark so we don't end up with two - # question marks. - data = data[:-1] self.soup.handle_data(data) self.soup.endData(ProcessingInstruction) @@ -128,15 +178,19 @@ def handle_pi(self, data): class HTMLParserTreeBuilder(HTMLTreeBuilder): is_xml = False - features = [HTML, STRICT, HTMLPARSER] + picklable = True + NAME = HTMLPARSER + features = [NAME, HTML, STRICT] def __init__(self, *args, **kwargs): - if CONSTRUCTOR_TAKES_STRICT: + if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED: kwargs['strict'] = False + if CONSTRUCTOR_TAKES_CONVERT_CHARREFS: + kwargs['convert_charrefs'] = False self.parser_args = (args, kwargs) def prepare_markup(self, markup, user_specified_encoding=None, - document_declared_encoding=None): + document_declared_encoding=None, exclude_encodings=None): """ :return: A 4-tuple (markup, original encoding, encoding declared within markup, whether any characters had to be @@ -147,7 +201,8 @@ def prepare_markup(self, markup, user_specified_encoding=None, return try_encodings = [user_specified_encoding, document_declared_encoding] - dammit = UnicodeDammit(markup, try_encodings, is_html=True) + dammit = UnicodeDammit(markup, try_encodings, is_html=True, + exclude_encodings=exclude_encodings) yield (dammit.markup, dammit.original_encoding, dammit.declared_html_encoding, dammit.contains_replacement_characters) @@ -162,6 +217,7 @@ def feed(self, markup): warnings.warn(RuntimeWarning( "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help.")) raise e + parser.already_closed_empty_element = [] # Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some # 3.2.3 code. This ensures they don't treat markup like <p></p> as a diff --git a/Contents/Libraries/Shared/bs4/builder/_lxml.py b/Contents/Libraries/Shared/bs4/builder/_lxml.py index fa5d49875..d2ca2872d 100644 --- a/Contents/Libraries/Shared/bs4/builder/_lxml.py +++ b/Contents/Libraries/Shared/bs4/builder/_lxml.py @@ -1,3 +1,5 @@ +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. __all__ = [ 'LXMLTreeBuilderForXML', 'LXMLTreeBuilder', @@ -7,7 +9,13 @@ from StringIO import StringIO import collections from lxml import etree -from bs4.element import Comment, Doctype, NamespacedAttribute +from bs4.element import ( + Comment, + Doctype, + NamespacedAttribute, + ProcessingInstruction, + XMLProcessingInstruction, +) from bs4.builder import ( FAST, HTML, @@ -24,9 +32,13 @@ class LXMLTreeBuilderForXML(TreeBuilder): DEFAULT_PARSER_CLASS = etree.XMLParser is_xml = True + processing_instruction_class = XMLProcessingInstruction + + NAME = "lxml-xml" + ALTERNATE_NAMES = ["xml"] # Well, it's permissive by XML parser standards. - features = [LXML, XML, FAST, PERMISSIVE] + features = [NAME, LXML, XML, FAST, PERMISSIVE] CHUNK_SIZE = 512 @@ -70,6 +82,7 @@ def _getNsTag(self, tag): return (None, tag) def prepare_markup(self, markup, user_specified_encoding=None, + exclude_encodings=None, document_declared_encoding=None): """ :yield: A series of 4-tuples. @@ -78,6 +91,16 @@ def prepare_markup(self, markup, user_specified_encoding=None, Each 4-tuple represents a strategy for parsing the document. """ + # Instead of using UnicodeDammit to convert the bytestring to + # Unicode using different encodings, use EncodingDetector to + # iterate over the encodings, and tell lxml to try to parse + # the document as each one in turn. + is_html = not self.is_xml + if is_html: + self.processing_instruction_class = ProcessingInstruction + else: + self.processing_instruction_class = XMLProcessingInstruction + if isinstance(markup, unicode): # We were given Unicode. Maybe lxml can parse Unicode on # this system? @@ -89,13 +112,9 @@ def prepare_markup(self, markup, user_specified_encoding=None, yield (markup.encode("utf8"), "utf8", document_declared_encoding, False) - # Instead of using UnicodeDammit to convert the bytestring to - # Unicode using different encodings, use EncodingDetector to - # iterate over the encodings, and tell lxml to try to parse - # the document as each one in turn. - is_html = not self.is_xml try_encodings = [user_specified_encoding, document_declared_encoding] - detector = EncodingDetector(markup, try_encodings, is_html) + detector = EncodingDetector( + markup, try_encodings, is_html, exclude_encodings) for encoding in detector.encodings: yield (detector.markup, encoding, document_declared_encoding, False) @@ -189,7 +208,9 @@ def end(self, name): self.nsmaps.pop() def pi(self, target, data): - pass + self.soup.endData() + self.soup.handle_data(target + ' ' + data) + self.soup.endData(self.processing_instruction_class) def data(self, content): self.soup.handle_data(content) @@ -212,8 +233,12 @@ def test_fragment_to_document(self, fragment): class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML): - features = [LXML, HTML, FAST, PERMISSIVE] + NAME = LXML + ALTERNATE_NAMES = ["lxml-html"] + + features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE] is_xml = False + processing_instruction_class = ProcessingInstruction def default_parser(self, encoding): return etree.HTMLParser diff --git a/Contents/Libraries/Shared/bs4/dammit.py b/Contents/Libraries/Shared/bs4/dammit.py index 59640b7ce..7965565f5 100644 --- a/Contents/Libraries/Shared/bs4/dammit.py +++ b/Contents/Libraries/Shared/bs4/dammit.py @@ -3,9 +3,12 @@ This library converts a bytestream to Unicode through any means necessary. It is heavily based on code from Mark Pilgrim's Universal -Feed Parser. It works best on XML and XML, but it does not rewrite the +Feed Parser. It works best on XML and HTML, but it does not rewrite the XML or HTML to reflect a new encoding; that's the tree builder's job. """ +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +__license__ = "MIT" import codecs from htmlentitydefs import codepoint2name @@ -212,8 +215,11 @@ class EncodingDetector: 5. Windows-1252. """ - def __init__(self, markup, override_encodings=None, is_html=False): + def __init__(self, markup, override_encodings=None, is_html=False, + exclude_encodings=None): self.override_encodings = override_encodings or [] + exclude_encodings = exclude_encodings or [] + self.exclude_encodings = set([x.lower() for x in exclude_encodings]) self.chardet_encoding = None self.is_html = is_html self.declared_encoding = None @@ -224,6 +230,8 @@ def __init__(self, markup, override_encodings=None, is_html=False): def _usable(self, encoding, tried): if encoding is not None: encoding = encoding.lower() + if encoding in self.exclude_encodings: + return False if encoding not in tried: tried.add(encoding) return True @@ -266,6 +274,9 @@ def encodings(self): def strip_byte_order_mark(cls, data): """If a byte-order mark is present, strip it and return the encoding it implies.""" encoding = None + if isinstance(data, unicode): + # Unicode data cannot have a byte-order mark. + return data, encoding if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \ and (data[2:4] != '\x00\x00'): encoding = 'utf-16be' @@ -299,14 +310,14 @@ def find_declared_encoding(cls, markup, is_html=False, search_entire_document=Fa else: xml_endpos = 1024 html_endpos = max(2048, int(len(markup) * 0.05)) - + declared_encoding = None declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos) if not declared_encoding_match and is_html: declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos) if declared_encoding_match is not None: declared_encoding = declared_encoding_match.groups()[0].decode( - 'ascii') + 'ascii', 'replace') if declared_encoding: return declared_encoding.lower() return None @@ -331,13 +342,14 @@ class UnicodeDammit: ] def __init__(self, markup, override_encodings=[], - smart_quotes_to=None, is_html=False): + smart_quotes_to=None, is_html=False, exclude_encodings=[]): self.smart_quotes_to = smart_quotes_to self.tried_encodings = [] self.contains_replacement_characters = False self.is_html = is_html - - self.detector = EncodingDetector(markup, override_encodings, is_html) + self.log = logging.getLogger(__name__) + self.detector = EncodingDetector( + markup, override_encodings, is_html, exclude_encodings) # Short-circuit if the data is in Unicode to begin with. if isinstance(markup, unicode) or markup == '': @@ -365,9 +377,10 @@ def __init__(self, markup, override_encodings=[], if encoding != "ascii": u = self._convert_from(encoding, "replace") if u is not None: - logging.warning( + self.log.warning( "Some characters could not be decoded, and were " - "replaced with REPLACEMENT CHARACTER.") + "replaced with REPLACEMENT CHARACTER." + ) self.contains_replacement_characters = True break @@ -723,7 +736,7 @@ def _codec(self, charset): 0xde : b'\xc3\x9e', # Þ 0xdf : b'\xc3\x9f', # ß 0xe0 : b'\xc3\xa0', # à - 0xe1 : b'\xa1', # á + 0xe1 : b'\xa1', # á 0xe2 : b'\xc3\xa2', # â 0xe3 : b'\xc3\xa3', # ã 0xe4 : b'\xc3\xa4', # ä diff --git a/Contents/Libraries/Shared/bs4/diagnose.py b/Contents/Libraries/Shared/bs4/diagnose.py index 4d0b00afa..8768332f5 100644 --- a/Contents/Libraries/Shared/bs4/diagnose.py +++ b/Contents/Libraries/Shared/bs4/diagnose.py @@ -1,4 +1,9 @@ """Diagnostic functions, mainly for use when doing tech support.""" + +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +__license__ = "MIT" + import cProfile from StringIO import StringIO from HTMLParser import HTMLParser @@ -33,18 +38,28 @@ def diagnose(data): if 'lxml' in basic_parsers: basic_parsers.append(["lxml", "xml"]) - from lxml import etree - print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION)) + try: + from lxml import etree + print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION)) + except ImportError, e: + print ( + "lxml is not installed or couldn't be imported.") + if 'html5lib' in basic_parsers: - import html5lib - print "Found html5lib version %s" % html5lib.__version__ + try: + import html5lib + print "Found html5lib version %s" % html5lib.__version__ + except ImportError, e: + print ( + "html5lib is not installed or couldn't be imported.") if hasattr(data, 'read'): data = data.read() elif os.path.exists(data): print '"%s" looks like a filename. Reading data from the file.' % data - data = open(data).read() + with open(data) as fp: + data = fp.read() elif data.startswith("http:") or data.startswith("https:"): print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup." diff --git a/Contents/Libraries/Shared/bs4/element.py b/Contents/Libraries/Shared/bs4/element.py index da9afdf48..9ef75f814 100644 --- a/Contents/Libraries/Shared/bs4/element.py +++ b/Contents/Libraries/Shared/bs4/element.py @@ -1,5 +1,10 @@ +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +__license__ = "MIT" + import collections import re +import shlex import sys import warnings from bs4.dammit import EntitySubstitution @@ -96,6 +101,8 @@ class HTMLAwareEntitySubstitution(EntitySubstitution): preformatted_tags = set(["pre"]) + preserve_whitespace_tags = set(['pre', 'textarea']) + @classmethod def _substitute_if_appropriate(cls, ns, f): if (isinstance(ns, NavigableString) @@ -124,8 +131,8 @@ class PageElement(object): # to methods like encode() and prettify(): # # "html" - All Unicode characters with corresponding HTML entities - # are converted to those entities on output. - # "minimal" - Bare ampersands and angle brackets are converted to + # are converted to those entities on output. + # "minimal" - Bare ampersands and angle brackets are converted to # XML entities: & < > # None - The null formatter. Unicode characters are never # converted to entities. This is not recommended, but it's @@ -166,11 +173,19 @@ def _is_xml(self): This is used when mapping a formatter name ("minimal") to an appropriate function (one that performs entity-substitution on - the contents of <script> and <style> tags, or not). It's + the contents of <script> and <style> tags, or not). It can be inefficient, but it should be called very rarely. """ + if self.known_xml is not None: + # Most of the time we will have determined this when the + # document is parsed. + return self.known_xml + + # Otherwise, it's likely that this element was created by + # direct invocation of the constructor from within the user's + # Python code. if self.parent is None: - # This is the top-level object. It should have .is_xml set + # This is the top-level object. It should have .known_xml set # from tree creation. If not, take a guess--BS is usually # used on HTML markup. return getattr(self, 'is_xml', False) @@ -185,24 +200,40 @@ def _formatter_for_name(self, name): return self.HTML_FORMATTERS.get( name, HTMLAwareEntitySubstitution.substitute_xml) - def setup(self, parent=None, previous_element=None): + def setup(self, parent=None, previous_element=None, next_element=None, + previous_sibling=None, next_sibling=None): """Sets up the initial relations between this element and other elements.""" self.parent = parent + self.previous_element = previous_element if previous_element is not None: self.previous_element.next_element = self - self.next_element = None - self.previous_sibling = None - self.next_sibling = None - if self.parent is not None and self.parent.contents: - self.previous_sibling = self.parent.contents[-1] + + self.next_element = next_element + if self.next_element: + self.next_element.previous_element = self + + self.next_sibling = next_sibling + if self.next_sibling: + self.next_sibling.previous_sibling = self + + if (not previous_sibling + and self.parent is not None and self.parent.contents): + previous_sibling = self.parent.contents[-1] + + self.previous_sibling = previous_sibling + if previous_sibling: self.previous_sibling.next_sibling = self nextSibling = _alias("next_sibling") # BS3 previousSibling = _alias("previous_sibling") # BS3 def replace_with(self, replace_with): + if not self.parent: + raise ValueError( + "Cannot replace one element with another when the" + "element to be replaced is not part of a tree.") if replace_with is self: return if replace_with is self.parent: @@ -216,6 +247,10 @@ def replace_with(self, replace_with): def unwrap(self): my_parent = self.parent + if not self.parent: + raise ValueError( + "Cannot replace an element with its contents when that" + "element is not part of a tree.") my_index = self.parent.index(self) self.extract() for child in reversed(self.contents[:]): @@ -240,17 +275,20 @@ def extract(self): last_child = self._last_descendant() next_element = last_child.next_element - if self.previous_element is not None: + if (self.previous_element is not None and + self.previous_element is not next_element): self.previous_element.next_element = next_element - if next_element is not None: + if next_element is not None and next_element is not self.previous_element: next_element.previous_element = self.previous_element self.previous_element = None last_child.next_element = None self.parent = None - if self.previous_sibling is not None: + if (self.previous_sibling is not None + and self.previous_sibling is not self.next_sibling): self.previous_sibling.next_sibling = self.next_sibling - if self.next_sibling is not None: + if (self.next_sibling is not None + and self.next_sibling is not self.previous_sibling): self.next_sibling.previous_sibling = self.previous_sibling self.previous_sibling = self.next_sibling = None return self @@ -263,13 +301,15 @@ def _last_descendant(self, is_initialized=True, accept_self=True): last_child = self while isinstance(last_child, Tag) and last_child.contents: last_child = last_child.contents[-1] - if not accept_self and last_child == self: + if not accept_self and last_child is self: last_child = None return last_child # BS3: Not part of the API! _lastRecursiveChild = _last_descendant def insert(self, position, new_child): + if new_child is None: + raise ValueError("Cannot insert None into a tag.") if new_child is self: raise ValueError("Cannot insert a tag into itself.") if (isinstance(new_child, basestring) @@ -478,6 +518,10 @@ def _find_one(self, method, name, attrs, text, **kwargs): def _find_all(self, name, attrs, text, limit, generator, **kwargs): "Iterates over a generator looking for things that match." + if text is None and 'string' in kwargs: + text = kwargs['string'] + del kwargs['string'] + if isinstance(name, SoupStrainer): strainer = name else: @@ -491,9 +535,16 @@ def _find_all(self, name, attrs, text, limit, generator, **kwargs): return ResultSet(strainer, result) elif isinstance(name, basestring): # Optimization to find all tags with a given name. + if name.count(':') == 1: + # This is a name with a prefix. + prefix, name = name.split(':', 1) + else: + prefix = None result = (element for element in generator if isinstance(element, Tag) - and element.name == name) + and element.name == name + and (prefix is None or element.prefix == prefix) + ) return ResultSet(strainer, result) results = ResultSet(strainer) while True: @@ -548,17 +599,17 @@ def parents(self): # Methods for supporting CSS selectors. - tag_name_re = re.compile('^[a-z0-9]+$') + tag_name_re = re.compile('^[a-zA-Z0-9][-.a-zA-Z0-9:_]*$') - # /^(\w+)\[(\w+)([=~\|\^\$\*]?)=?"?([^\]"]*)"?\]$/ - # \---/ \---/\-------------/ \-------/ - # | | | | - # | | | The value - # | | ~,|,^,$,* or = - # | Attribute + # /^([a-zA-Z0-9][-.a-zA-Z0-9:_]*)\[(\w+)([=~\|\^\$\*]?)=?"?([^\]"]*)"?\]$/ + # \---------------------------/ \---/\-------------/ \-------/ + # | | | | + # | | | The value + # | | ~,|,^,$,* or = + # | Attribute # Tag attribselect_re = re.compile( - r'^(?P<tag>\w+)?\[(?P<attribute>\w+)(?P<operator>[=~\|\^\$\*]?)' + + r'^(?P<tag>[a-zA-Z0-9][-.a-zA-Z0-9:_]*)?\[(?P<attribute>[\w-]+)(?P<operator>[=~\|\^\$\*]?)' + r'=?"?(?P<value>[^\]"]*)"?\]$' ) @@ -605,7 +656,7 @@ def _includes_value(element): return lambda el: el._attr_value_as_string( attribute, '').startswith(value) elif operator == '$': - # string represenation of `attribute` ends with `value` + # string representation of `attribute` ends with `value` return lambda el: el._attr_value_as_string( attribute, '').endswith(value) elif operator == '*': @@ -645,6 +696,11 @@ class NavigableString(unicode, PageElement): PREFIX = '' SUFFIX = '' + # We can't tell just by looking at a string whether it's contained + # in an XML document or an HTML document. + + known_xml = None + def __new__(cls, value): """Create a new NavigableString. @@ -654,11 +710,17 @@ def __new__(cls, value): how to handle non-ASCII characters. """ if isinstance(value, unicode): - return unicode.__new__(cls, value) - return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING) + u = unicode.__new__(cls, value) + else: + u = unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING) + u.setup() + return u def __copy__(self): - return self + """A copy of a NavigableString has the same contents and class + as the original, but it is not connected to the parse tree. + """ + return type(self)(self) def __getnewargs__(self): return (unicode(self),) @@ -705,7 +767,13 @@ class CData(PreformattedString): SUFFIX = u']]>' class ProcessingInstruction(PreformattedString): + """A SGML processing instruction.""" + PREFIX = u'<?' + SUFFIX = u'>' + +class XMLProcessingInstruction(ProcessingInstruction): + """An XML processing instruction.""" PREFIX = u'<?' SUFFIX = u'?>' @@ -716,8 +784,8 @@ class Comment(PreformattedString): class Declaration(PreformattedString): - PREFIX = u'<!' - SUFFIX = u'!>' + PREFIX = u'<?' + SUFFIX = u'?>' class Doctype(PreformattedString): @@ -743,7 +811,8 @@ class Tag(PageElement): """Represents a found HTML tag with its attributes and contents.""" def __init__(self, parser=None, builder=None, name=None, namespace=None, - prefix=None, attrs=None, parent=None, previous=None): + prefix=None, attrs=None, parent=None, previous=None, + is_xml=None): "Basic constructor." if parser is None: @@ -757,13 +826,31 @@ def __init__(self, parser=None, builder=None, name=None, namespace=None, self.name = name self.namespace = namespace self.prefix = prefix + if builder is not None: + preserve_whitespace_tags = builder.preserve_whitespace_tags + else: + if is_xml: + preserve_whitespace_tags = [] + else: + preserve_whitespace_tags = HTMLAwareEntitySubstitution.preserve_whitespace_tags + self.preserve_whitespace_tags = preserve_whitespace_tags if attrs is None: attrs = {} - elif attrs and builder.cdata_list_attributes: - attrs = builder._replace_cdata_list_attribute_values( - self.name, attrs) + elif attrs: + if builder is not None and builder.cdata_list_attributes: + attrs = builder._replace_cdata_list_attribute_values( + self.name, attrs) + else: + attrs = dict(attrs) else: attrs = dict(attrs) + + # If possible, determine ahead of time whether this tag is an + # XML tag. + if builder: + self.known_xml = builder.is_xml + else: + self.known_xml = is_xml self.attrs = attrs self.contents = [] self.setup(parent, previous) @@ -778,6 +865,18 @@ def __init__(self, parser=None, builder=None, name=None, namespace=None, parserClass = _alias("parser_class") # BS3 + def __copy__(self): + """A copy of a Tag is a new Tag, unconnected to the parse tree. + Its contents are a copy of the old Tag's contents. + """ + clone = type(self)(None, self.builder, self.name, self.namespace, + self.prefix, self.attrs, is_xml=self._is_xml) + for attr in ('can_be_empty_element', 'hidden'): + setattr(clone, attr, getattr(self, attr)) + for child in self.contents: + clone.append(child.__copy__()) + return clone + @property def is_empty_element(self): """Is this tag an empty-element tag? (aka a self-closing tag) @@ -893,6 +992,13 @@ def get(self, key, default=None): attribute.""" return self.attrs.get(key, default) + def get_attribute_list(self, key, default=None): + """The same as get(), but always returns a list.""" + value = self.get(key, default) + if not isinstance(value, list): + value = [value] + return value + def has_attr(self, key): return key in self.attrs @@ -944,7 +1050,7 @@ def __getattr__(self, tag): tag_name, tag_name)) return self.find(tag_name) # We special case contents to avoid recursion. - elif not tag.startswith("__") and not tag=="contents": + elif not tag.startswith("__") and not tag == "contents": return self.find(tag) raise AttributeError( "'%s' object has no attribute '%s'" % (self.__class__, tag)) @@ -971,15 +1077,25 @@ def __ne__(self, other): as defined in __eq__.""" return not self == other - def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING): + def __repr__(self, encoding="unicode-escape"): """Renders this tag as a string.""" - return self.encode(encoding) + if PY3K: + # "The return value must be a string object", i.e. Unicode + return self.decode() + else: + # "The return value must be a string object", i.e. a bytestring. + # By convention, the return value of __repr__ should also be + # an ASCII string. + return self.encode(encoding) def __unicode__(self): return self.decode() def __str__(self): - return self.encode() + if PY3K: + return self.decode() + else: + return self.encode() if PY3K: __str__ = __repr__ = __unicode__ @@ -994,10 +1110,11 @@ def encode(self, encoding=DEFAULT_OUTPUT_ENCODING, def _should_pretty_print(self, indent_level): """Should this tag be pretty-printed?""" + return ( - indent_level is not None and - (self.name not in HTMLAwareEntitySubstitution.preformatted_tags - or self._is_xml)) + indent_level is not None + and self.name not in self.preserve_whitespace_tags + ) def decode(self, indent_level=None, eventual_encoding=DEFAULT_OUTPUT_ENCODING, @@ -1103,12 +1220,18 @@ def decode_contents(self, indent_level=None, formatter="minimal"): """Renders the contents of this tag as a Unicode string. + :param indent_level: Each line of the rendering will be + indented this many spaces. + :param eventual_encoding: The tag is destined to be encoded into this encoding. This method is _not_ responsible for performing that encoding. This information is passed in so that it can be substituted in if the document contains a <META> tag that mentions the document's encoding. + + :param formatter: The output formatter responsible for converting + entities to Unicode characters. """ # First off, turn a string formatter into a function. This # will stop the lookup from happening over and over again. @@ -1137,7 +1260,17 @@ def decode_contents(self, indent_level=None, def encode_contents( self, indent_level=None, encoding=DEFAULT_OUTPUT_ENCODING, formatter="minimal"): - """Renders the contents of this tag as a bytestring.""" + """Renders the contents of this tag as a bytestring. + + :param indent_level: Each line of the rendering will be + indented this many spaces. + + :param eventual_encoding: The bytestring will be in this encoding. + + :param formatter: The output formatter responsible for converting + entities to Unicode characters. + """ + contents = self.decode_contents(indent_level, encoding, formatter) return contents.encode(encoding) @@ -1201,26 +1334,57 @@ def descendants(self): _selector_combinators = ['>', '+', '~'] _select_debug = False - def select(self, selector, _candidate_generator=None): + quoted_colon = re.compile('"[^"]*:[^"]*"') + def select_one(self, selector): + """Perform a CSS selection operation on the current element.""" + value = self.select(selector, limit=1) + if value: + return value[0] + return None + + def select(self, selector, _candidate_generator=None, limit=None): """Perform a CSS selection operation on the current element.""" - tokens = selector.split() + + # Handle grouping selectors if ',' exists, ie: p,a + if ',' in selector: + context = [] + for partial_selector in selector.split(','): + partial_selector = partial_selector.strip() + if partial_selector == '': + raise ValueError('Invalid group selection syntax: %s' % selector) + candidates = self.select(partial_selector, limit=limit) + for candidate in candidates: + if candidate not in context: + context.append(candidate) + + if limit and len(context) >= limit: + break + return context + tokens = shlex.split(selector) current_context = [self] if tokens[-1] in self._selector_combinators: raise ValueError( 'Final combinator "%s" is missing an argument.' % tokens[-1]) + if self._select_debug: print 'Running CSS selector "%s"' % selector + for index, token in enumerate(tokens): - if self._select_debug: - print ' Considering token "%s"' % token - recursive_candidate_generator = None - tag_name = None + new_context = [] + new_context_ids = set([]) + if tokens[index-1] in self._selector_combinators: # This token was consumed by the previous combinator. Skip it. if self._select_debug: print ' Token was consumed by the previous combinator.' continue + + if self._select_debug: + print ' Considering token "%s"' % token + recursive_candidate_generator = None + tag_name = None + # Each operation corresponds to a checker function, a rule # for determining whether a candidate matches the # selector. Candidates are generated by the active @@ -1248,7 +1412,7 @@ def classes_match(candidate): return classes.issubset(candidate.get('class', [])) checker = classes_match - elif ':' in token: + elif ':' in token and not self.quoted_colon.search(token): # Pseudo-class tag_name, pseudo = token.split(':', 1) if tag_name == '': @@ -1256,35 +1420,35 @@ def classes_match(candidate): "A pseudo-class must be prefixed with a tag name.") pseudo_attributes = re.match('([a-zA-Z\d-]+)\(([a-zA-Z\d]+)\)', pseudo) found = [] - if pseudo_attributes is not None: + if pseudo_attributes is None: + pseudo_type = pseudo + pseudo_value = None + else: pseudo_type, pseudo_value = pseudo_attributes.groups() - if pseudo_type == 'nth-of-type': - try: - pseudo_value = int(pseudo_value) - except: - raise NotImplementedError( - 'Only numeric values are currently supported for the nth-of-type pseudo-class.') - if pseudo_value < 1: - raise ValueError( - 'nth-of-type pseudo-class value must be at least 1.') - class Counter(object): - def __init__(self, destination): - self.count = 0 - self.destination = destination - - def nth_child_of_type(self, tag): - self.count += 1 - if self.count == self.destination: - return True - if self.count > self.destination: - # Stop the generator that's sending us - # these things. - raise StopIteration() - return False - checker = Counter(pseudo_value).nth_child_of_type - else: + if pseudo_type == 'nth-of-type': + try: + pseudo_value = int(pseudo_value) + except: raise NotImplementedError( - 'Only the following pseudo-classes are implemented: nth-of-type.') + 'Only numeric values are currently supported for the nth-of-type pseudo-class.') + if pseudo_value < 1: + raise ValueError( + 'nth-of-type pseudo-class value must be at least 1.') + class Counter(object): + def __init__(self, destination): + self.count = 0 + self.destination = destination + + def nth_child_of_type(self, tag): + self.count += 1 + if self.count == self.destination: + return True + else: + return False + checker = Counter(pseudo_value).nth_child_of_type + else: + raise NotImplementedError( + 'Only the following pseudo-classes are implemented: nth-of-type.') elif token == '*': # Star selector -- matches everything @@ -1311,7 +1475,6 @@ def next_tag_sibling(tag): else: raise ValueError( 'Unsupported or invalid CSS selector: "%s"' % token) - if recursive_candidate_generator: # This happens when the selector looks like "> foo". # @@ -1361,8 +1524,7 @@ def default_candidate_generator(tag): else: _use_candidate_generator = _candidate_generator - new_context = [] - new_context_ids = set([]) + count = 0 for tag in current_context: if self._select_debug: print " Running candidate generator on %s %s" % ( @@ -1391,6 +1553,8 @@ def default_candidate_generator(tag): print " FAILURE %s %s" % (candidate.name, repr(candidate.attrs)) current_context = new_context + if limit and len(current_context) >= limit: + current_context = current_context[:limit] if self._select_debug: print "Final verdict:" @@ -1548,28 +1712,22 @@ def search(self, markup): "I don't know how to match against a %s" % markup.__class__) return found - def _matches(self, markup, match_against): + def _matches(self, markup, match_against, already_tried=None): # print u"Matching %s against %s" % (markup, match_against) result = False if isinstance(markup, list) or isinstance(markup, tuple): # This should only happen when searching a multi-valued attribute # like 'class'. - if (isinstance(match_against, unicode) - and ' ' in match_against): - # A bit of a special case. If they try to match "foo - # bar" on a multivalue attribute's value, only accept - # the literal value "foo bar" - # - # XXX This is going to be pretty slow because we keep - # splitting match_against. But it shouldn't come up - # too often. - return (whitespace_re.split(match_against) == markup) - else: - for item in markup: - if self._matches(item, match_against): - return True - return False - + for item in markup: + if self._matches(item, match_against): + return True + # We didn't match any particular value of the multivalue + # attribute, but maybe we match the attribute value when + # considered as a string. + if self._matches(' '.join(markup), match_against): + return True + return False + if match_against is True: # True matches any non-None value. return markup is not None @@ -1579,6 +1737,7 @@ def _matches(self, markup, match_against): # Custom callables take the tag as an argument, but all # other ways of matching match the tag name as a string. + original_markup = markup if isinstance(markup, Tag): markup = markup.name @@ -1589,18 +1748,51 @@ def _matches(self, markup, match_against): # None matches None, False, an empty string, an empty list, and so on. return not match_against - if isinstance(match_against, unicode): + if (hasattr(match_against, '__iter__') + and not isinstance(match_against, basestring)): + # We're asked to match against an iterable of items. + # The markup must be match at least one item in the + # iterable. We'll try each one in turn. + # + # To avoid infinite recursion we need to keep track of + # items we've already seen. + if not already_tried: + already_tried = set() + for item in match_against: + if item.__hash__: + key = item + else: + key = id(item) + if key in already_tried: + continue + else: + already_tried.add(key) + if self._matches(original_markup, item, already_tried): + return True + else: + return False + + # Beyond this point we might need to run the test twice: once against + # the tag's name and once against its prefixed name. + match = False + + if not match and isinstance(match_against, unicode): # Exact string match - return markup == match_against + match = markup == match_against - if hasattr(match_against, 'match'): + if not match and hasattr(match_against, 'search'): # Regexp match return match_against.search(markup) - if hasattr(match_against, '__iter__'): - # The markup must be an exact match against something - # in the iterable. - return markup in match_against + if (not match + and isinstance(original_markup, Tag) + and original_markup.prefix): + # Try the whole thing again with the prefixed tag name. + return self._matches( + original_markup.prefix + ':' + original_markup.name, match_against + ) + + return match class ResultSet(list): @@ -1609,3 +1801,8 @@ class ResultSet(list): def __init__(self, source, result=()): super(ResultSet, self).__init__(result) self.source = source + + def __getattr__(self, key): + raise AttributeError( + "ResultSet object has no attribute '%s'. You're probably treating a list of items like a single item. Did you call find_all() when you meant to call find()?" % key + ) diff --git a/Contents/Libraries/Shared/bs4/testing.py b/Contents/Libraries/Shared/bs4/testing.py index fd4495ac5..6ba2506c4 100644 --- a/Contents/Libraries/Shared/bs4/testing.py +++ b/Contents/Libraries/Shared/bs4/testing.py @@ -1,5 +1,10 @@ """Helper classes for tests.""" +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +__license__ = "MIT" + +import pickle import copy import functools import unittest @@ -43,6 +48,16 @@ def assertSoupEquals(self, to_parse, compare_parsed_to=None): self.assertEqual(obj.decode(), self.document_for(compare_parsed_to)) + def assertConnectedness(self, element): + """Ensure that next_element and previous_element are properly + set for all descendants of the given element. + """ + earlier = None + for e in element.descendants: + if earlier: + self.assertEqual(e, earlier.next_element) + self.assertEqual(earlier, e.previous_element) + earlier = e class HTMLTreeBuilderSmokeTest(object): @@ -54,6 +69,27 @@ class HTMLTreeBuilderSmokeTest(object): markup in these tests, there's not much room for interpretation. """ + def test_empty_element_tags(self): + """Verify that all HTML4 and HTML5 empty element (aka void element) tags + are handled correctly. + """ + for name in [ + 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr', + 'spacer', 'frame' + ]: + soup = self.soup("") + new_tag = soup.new_tag(name) + self.assertEqual(True, new_tag.is_empty_element) + + def test_pickle_and_unpickle_identity(self): + # Pickling a tree, then unpickling it, yields a tree identical + # to the original. + tree = self.soup("<a><b>foo</a>") + dumped = pickle.dumps(tree, 2) + loaded = pickle.loads(dumped) + self.assertEqual(loaded.__class__, BeautifulSoup) + self.assertEqual(loaded.decode(), tree.decode()) + def assertDoctypeHandled(self, doctype_fragment): """Assert that a given doctype string is handled correctly.""" doctype_str, soup = self._document_with_doctype(doctype_fragment) @@ -114,6 +150,19 @@ def test_real_xhtml_document(self): soup.encode("utf-8").replace(b"\n", b""), markup.replace(b"\n", b"")) + def test_processing_instruction(self): + # We test both Unicode and bytestring to verify that + # process_markup correctly sets processing_instruction_class + # even when the markup is already Unicode and there is no + # need to process anything. + markup = u"""<?PITarget PIContent?>""" + soup = self.soup(markup) + self.assertEqual(markup, soup.decode()) + + markup = b"""<?PITarget PIContent?>""" + soup = self.soup(markup) + self.assertEqual(markup, soup.encode("utf8")) + def test_deepcopy(self): """Make sure you can copy the tree builder. @@ -155,6 +204,23 @@ def test_br_is_always_empty_element_tag(self): def test_nested_formatting_elements(self): self.assertSoupEquals("<em><em></em></em>") + def test_double_head(self): + html = '''<!DOCTYPE html> +<html> +<head> +<title>Ordinary HEAD element test + + + +Hello, world! + + +''' + soup = self.soup(html) + self.assertEqual("text/javascript", soup.find('script')['type']) + def test_comment(self): # Comments are represented as Comment objects. markup = "

foobaz

" @@ -171,9 +237,22 @@ def test_comment(self): self.assertEqual(comment, baz.previous_element) def test_preserved_whitespace_in_pre_and_textarea(self): - """Whitespace must be preserved in
 and ")
+        """Whitespace must be preserved in 
 and "
+        self.assertSoupEquals(pre_markup)
+        self.assertSoupEquals(textarea_markup)
+
+        soup = self.soup(pre_markup)
+        self.assertEqual(soup.pre.prettify(), pre_markup)
+
+        soup = self.soup(textarea_markup)
+        self.assertEqual(soup.textarea.prettify(), textarea_markup)
+
+        soup = self.soup("")
+        self.assertEqual(soup.textarea.prettify(), "")
 
     def test_nested_inline_elements(self):
         """Inline elements can be nested indefinitely."""
@@ -221,6 +300,14 @@ def test_deeply_nested_multivalued_attribute(self):
         soup = self.soup(markup)
         self.assertEqual(["css"], soup.div.div['class'])
 
+    def test_multivalued_attribute_on_html(self):
+        # html5lib uses a different API to set the attributes ot the
+        #  tag. This has caused problems with multivalued
+        # attributes.
+        markup = ''
+        soup = self.soup(markup)
+        self.assertEqual(["a", "b"], soup.html['class'])
+
     def test_angle_brackets_in_attribute_values_are_escaped(self):
         self.assertSoupEquals('', '')
 
@@ -253,6 +340,42 @@ def test_multipart_strings(self):
         soup = self.soup("

\nfoo

") self.assertEqual("p", soup.h2.string.next_element.name) self.assertEqual("p", soup.p.name) + self.assertConnectedness(soup) + + def test_empty_element_tags(self): + """Verify consistent handling of empty-element tags, + no matter how they come in through the markup. + """ + self.assertSoupEquals('


', "


") + self.assertSoupEquals('


', "


") + + def test_head_tag_between_head_and_body(self): + "Prevent recurrence of a bug in the html5lib treebuilder." + content = """ + + foo + +""" + soup = self.soup(content) + self.assertNotEqual(None, soup.html.body) + self.assertConnectedness(soup) + + def test_multiple_copies_of_a_tag(self): + "Prevent recurrence of a bug in the html5lib treebuilder." + content = """ + + + + + +""" + soup = self.soup(content) + self.assertConnectedness(soup.article) def test_basic_namespaces(self): """Parsers don't need to *understand* namespaces, but at the @@ -399,7 +522,9 @@ def test_real_hebrew_document(self): hebrew_document = b'Hebrew (ISO 8859-8) in Visual Directionality

Hebrew (ISO 8859-8) in Visual Directionality

\xed\xe5\xec\xf9' soup = self.soup( hebrew_document, from_encoding="iso8859-8") - self.assertEqual(soup.original_encoding, 'iso8859-8') + # Some tree builders call it iso8859-8, others call it iso-8859-9. + # That's not a difference we really care about. + assert soup.original_encoding in ('iso8859-8', 'iso-8859-8') self.assertEqual( soup.encode('utf-8'), hebrew_document.decode("iso8859-8").encode("utf-8")) @@ -463,11 +588,30 @@ def test_tag_with_no_attributes_can_have_attributes_added(self): class XMLTreeBuilderSmokeTest(object): + def test_pickle_and_unpickle_identity(self): + # Pickling a tree, then unpickling it, yields a tree identical + # to the original. + tree = self.soup("foo") + dumped = pickle.dumps(tree, 2) + loaded = pickle.loads(dumped) + self.assertEqual(loaded.__class__, BeautifulSoup) + self.assertEqual(loaded.decode(), tree.decode()) + def test_docstring_generated(self): soup = self.soup("") self.assertEqual( soup.encode(), b'\n') + def test_xml_declaration(self): + markup = b"""\n""" + soup = self.soup(markup) + self.assertEqual(markup, soup.encode("utf8")) + + def test_processing_instruction(self): + markup = b"""\n""" + soup = self.soup(markup) + self.assertEqual(markup, soup.encode("utf8")) + def test_real_xhtml_document(self): """A real XHTML document should come out *exactly* the same as it went in.""" markup = b""" @@ -485,7 +629,7 @@ def test_formatter_processes_script_tag_for_xml_documents(self): """ - soup = BeautifulSoup(doc, "xml") + soup = BeautifulSoup(doc, "lxml-xml") # lxml would have stripped this while parsing, but we can add # it later. soup.script.string = 'console.log("< < hey > > ");' @@ -544,6 +688,40 @@ def test_namespaced_attributes_xml_namespace(self): soup = self.soup(markup) self.assertEqual(unicode(soup.foo), markup) + def test_find_by_prefixed_name(self): + doc = """ +foo + bar + baz + +""" + soup = self.soup(doc) + + # There are three tags. + self.assertEqual(3, len(soup.find_all('tag'))) + + # But two of them are ns1:tag and one of them is ns2:tag. + self.assertEqual(2, len(soup.find_all('ns1:tag'))) + self.assertEqual(1, len(soup.find_all('ns2:tag'))) + + self.assertEqual(1, len(soup.find_all('ns2:tag', key='value'))) + self.assertEqual(3, len(soup.find_all(['ns1:tag', 'ns2:tag']))) + + def test_copy_tag_preserves_namespace(self): + xml = """ +""" + + soup = self.soup(xml) + tag = soup.document + duplicate = copy.copy(tag) + + # The two tags have the same namespace prefix. + self.assertEqual(tag.prefix, duplicate.prefix) + + class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest): """Smoke test for a tree builder that supports HTML5.""" diff --git a/Contents/Libraries/Shared/bs4/tests/test_builder_registry.py b/Contents/Libraries/Shared/bs4/tests/test_builder_registry.py index 92ad10fb0..90cad8293 100644 --- a/Contents/Libraries/Shared/bs4/tests/test_builder_registry.py +++ b/Contents/Libraries/Shared/bs4/tests/test_builder_registry.py @@ -1,6 +1,7 @@ """Tests of the builder registry.""" import unittest +import warnings from bs4 import BeautifulSoup from bs4.builder import ( @@ -67,10 +68,15 @@ def test_named_library(self): HTMLParserTreeBuilder) def test_beautifulsoup_constructor_does_lookup(self): - # You can pass in a string. - BeautifulSoup("", features="html") - # Or a list of strings. - BeautifulSoup("", features=["html", "fast"]) + + with warnings.catch_warnings(record=True) as w: + # This will create a warning about not explicitly + # specifying a parser, but we'll ignore it. + + # You can pass in a string. + BeautifulSoup("", features="html") + # Or a list of strings. + BeautifulSoup("", features=["html", "fast"]) # You'll get an exception if BS can't find an appropriate # builder. diff --git a/Contents/Libraries/Shared/bs4/tests/test_html5lib.py b/Contents/Libraries/Shared/bs4/tests/test_html5lib.py index 594c3e1f2..0f89d6244 100644 --- a/Contents/Libraries/Shared/bs4/tests/test_html5lib.py +++ b/Contents/Libraries/Shared/bs4/tests/test_html5lib.py @@ -83,3 +83,48 @@ def test_reparented_markup_ends_with_whitespace(self): soup = self.soup(markup) self.assertEqual(u"

foo

\n

bar

\n", soup.body.decode()) self.assertEqual(2, len(soup.find_all('p'))) + + def test_reparented_markup_containing_identical_whitespace_nodes(self): + """Verify that we keep the two whitespace nodes in this + document distinct when reparenting the adjacent tags. + """ + markup = '
' + soup = self.soup(markup) + space1, space2 = soup.find_all(string=' ') + tbody1, tbody2 = soup.find_all('tbody') + assert space1.next_element is tbody1 + assert tbody2.next_element is space2 + + def test_reparented_markup_containing_children(self): + markup = '' + soup = self.soup(markup) + noscript = soup.noscript + self.assertEqual("target", noscript.next_element) + target = soup.find(string='target') + + # The 'aftermath' string was duplicated; we want the second one. + final_aftermath = soup.find_all(string='aftermath')[-1] + + # The