diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 5c082e9e..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Java Gradle CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-java/ for more details -# -version: 2 -jobs: - build: - docker: - # specify the version you desire here - - image: circleci/openjdk:11-jdk - - # Specify service dependencies here if necessary - # CircleCI maintains a library of pre-built images - # documented at https://circleci.com/docs/2.0/circleci-images/ - # - image: circleci/postgres:9.4 - - working_directory: ~/repo - - environment: - # Customize the JVM maximum heap limit - JVM_OPTS: -Xmx3200m - TERM: dumb - - steps: - - checkout - - # Download and cache dependencies - - restore_cache: - keys: - - v1-dependencies-{{ checksum "build.gradle" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - - run: ./gradlew dependencies - - - save_cache: - paths: - - ~/.gradle - key: v1-dependencies-{{ checksum "build.gradle" }} - - # run tests! - - run: ./gradlew test jacocoTestReport coveralls --no-daemon - - - store_test_results: - path: build/reports/jacoco/test/jacocoTestReport.xml - - store_test_results: - path: build/reports/tests/test - - store_artifacts: - path: build/reports/jacoco/test/jacocoTestReport.xml - - store_artifacts: - path: build/reports/tests/test - - store_artifacts: - path: build/reports/jacoco/test/html/index.html - - store_artifacts: - path: build/reports/tests/test/index.html \ No newline at end of file diff --git a/.github/workflows/ci-build-release.yml b/.github/workflows/ci-build-release.yml index 0522dd7f..4ef3bf75 100644 --- a/.github/workflows/ci-build-release.yml +++ b/.github/workflows/ci-build-release.yml @@ -5,9 +5,7 @@ on: push: tags: - 'v*' - pull_request: - branches: [ main, master ] - + concurrency: group: docker cancel-in-progress: true @@ -17,17 +15,26 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: Set up JDK 11 - uses: actions/setup-java@v1 + - uses: actions/checkout@v3 + - name: Set up JDK 17 + uses: actions/setup-java@v4 with: - java-version: 1.11 + java-version: '17.0.10+7' + distribution: 'temurin' + cache: 'gradle' - name: Build with Gradle run: ./gradlew build -x test - name: Test with Gradle Jacoco and Coveralls run: ./gradlew test jacocoTestReport coveralls --no-daemon + - name: Coveralls GitHub Action + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + format: jacoco + + docker-build: needs: [build] runs-on: ubuntu-latest @@ -48,6 +55,8 @@ jobs: fi echo "TAGS=${TAGS}" echo ::set-output name=tags::${TAGS} + - name: Create more disk space + run: sudo rm -rf /usr/share/dotnet && sudo rm -rf /opt/ghc && sudo rm -rf "/usr/local/share/boost" && sudo rm -rf "$AGENT_TOOLSDIRECTORY" - uses: actions/checkout@v2 - name: Build and push id: docker_build diff --git a/.github/workflows/ci-build-unstable.yml b/.github/workflows/ci-build-unstable.yml index 8b4523cf..79ab3eda 100644 --- a/.github/workflows/ci-build-unstable.yml +++ b/.github/workflows/ci-build-unstable.yml @@ -12,36 +12,49 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: Set up JDK 11 - uses: actions/setup-java@v1 + - uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 with: - java-version: 1.11 + java-version: '17.0.10+7' + distribution: 'temurin' + cache: 'gradle' - name: Build with Gradle run: ./gradlew build -x test - name: Test with Gradle Jacoco and Coveralls run: ./gradlew test jacocoTestReport coveralls --no-daemon -# - name: Coveralls GitHub Action -# uses: coverallsapp/github-action@v1.0.1 -# with: -# github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Coveralls GitHub Action + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + format: jacoco - docker-build: - needs: [build] - runs-on: self-hosted + docker-build: + needs: [ build ] + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - name: Create more disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /opt/hostedtoolcache + - uses: actions/checkout@v4 + - name: Build and push + id: docker_build + uses: mr-smithers-excellent/docker-build-push@v6 with: - submodules: false - - name: set up proxy http - run: echo -e "systemProp.http.proxyHost=wwwout.nims.go.jp\nsystemProp.http.proxyPort=8888\nsystemProp.http.nonProxyHosts=*.nims.go.jp,localhost\n\nsystemProp.https.proxyHost=wwwout.nims.go.jp\nsystemProp.https.proxyPort=8888\nsystemProp.https.nonProxyHosts=*.nims.go.jp,localhost" >> gradle.properties - - name: check result proxy - run: cat gradle.properties - - name: Build the Docker image - run: docker build . --file Dockerfile --tag lfoppiano/grobid-superconductors:latest - - name: Cleanup older than 24h images and containers - run: docker system prune --filter "until=24h" --force + dockerfile: Dockerfile + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + image: lfoppiano/grobid-superconductors + registry: docker.io + pushImage: ${{ github.event_name != 'pull_request' }} + tags: latest-develop + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/ci-integration-manual.yml b/.github/workflows/ci-integration-manual.yml new file mode 100644 index 00000000..ee20f532 --- /dev/null +++ b/.github/workflows/ci-integration-manual.yml @@ -0,0 +1,32 @@ +name: Run integration tests manually + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout grobid home + uses: actions/checkout@v4 + with: + repository: kermitt2/grobid + path: ./grobid + - name: Checkout grobid-superconductors + uses: actions/checkout@v4 + with: + path: ./grobid/grobid-superconductors + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17.0.10+7' + distribution: 'temurin' + cache: 'gradle' + - name: Build and run integration tests + working-directory: ./grobid/grobid-superconductors + run: ./gradlew copyModels integration --no-daemon + diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index e3dde0b1..00000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,70 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ master, test-codeql ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] - schedule: - - cron: '37 14 * * 2' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'java' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 diff --git a/.java-version b/.java-version index 2dbc24b3..03b6389f 100644 --- a/.java-version +++ b/.java-version @@ -1 +1 @@ -11.0 +17.0 diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..476bfa1e --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,54 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: +- family-names: "Foppiano" + given-names: "Luca" + orcid: "https://orcid.org/0000-0002-6114-6164" +- family-names: "de Castro" + given-names: "Pedro Baptista" + orcid: "https://orcid.org/0000-0001-8673-2840" +- family-names: "Ortiz Suarez" + given-names: "Pedro" + orcid: "https://orcid.org/0000-0003-0343-8852" +- family-names: "Terashima" + given-names: "Kensei" + orcid: "https://orcid.org/0000-0003-0375-3043" +- family-names: "Takano" + given-names: "Yoshihiko" + orcid: "https://orcid.org/0000-0002-1541-6928" +- family-names: "Ishii" + given-names: "Masashi" + orcid: "https://orcid.org/0000-0003-0357-2832" +title: "grobid-superconductors" +version: 2.0.4 +url: "https://github.com/lfoppiano/grobid-superconductors" +preferred-citation: + type: article + authors: + - family-names: "Foppiano" + given-names: "Luca" + orcid: "https://orcid.org/0000-0002-6114-6164" + - family-names: "de Castro" + given-names: "Pedro Baptista" + orcid: "https://orcid.org/0000-0001-8673-2840" + - family-names: "Ortiz Suarez" + given-names: "Pedro" + orcid: "https://orcid.org/0000-0003-0343-8852" + - family-names: "Terashima" + given-names: "Kensei" + orcid: "https://orcid.org/0000-0003-0375-3043" + - family-names: "Takano" + given-names: "Yoshihiko" + orcid: "https://orcid.org/0000-0002-1541-6928" + - family-names: "Ishii" + given-names: "Masashi" + orcid: "https://orcid.org/0000-0003-0357-2832" + doi: "10.1080/27660400.2022.2153633" + journal: " Science and Technology of Advanced Materials: Methods" + month: 1 +# start: 1 # First page number +# end: 10 # Last page number + title: "Automatic extraction of materials and properties from superconductors scientific literature" + issue: 1 + volume: 3 + year: 2023 diff --git a/Dockerfile b/Dockerfile index 72df84a2..372a49b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,51 +21,42 @@ # build builder image # ------------------- -FROM openjdk:8u342-jdk as builder +FROM openjdk:17-jdk-slim as builder USER root RUN apt-get update && \ - apt-get -y --no-install-recommends install apt-utils libxml2 git - -RUN mkdir -p /opt/grobid-source/grobid-home/models - -WORKDIR /opt/grobid-source -COPY gradle.properties . - -RUN git clone --depth 1 --branch 0.7.2 https://github.com/kermitt2/grobid-quantities.git ./grobid-quantities && \ - cd grobid-quantities - -WORKDIR /opt/grobid-source/grobid-quantities -COPY gradle.properties . + apt-get -y --no-install-recommends install apt-utils libxml2 git unzip WORKDIR /opt/grobid-source -RUN mkdir -p grobid-superconductors/resources/config grobid-superconductors/resources/models grobid-superconductors/gradle grobid-superconductors/localLibs grobid-superconductors/resources/web grobid-superconductors/src - -COPY ./.git/ ./grobid-superconductors/.git -COPY resources/models/ ./grobid-superconductors/resources/models/ -COPY resources/config/ ./grobid-superconductors/resources/config/ -COPY gradle/ ./grobid-superconductors/gradle/ -COPY src/ ./grobid-superconductors/src/ -COPY localLibs/ ./grobid-superconductors/localLibs/ -COPY build.gradle ./grobid-superconductors/ -COPY settings.gradle ./grobid-superconductors/ -COPY gradlew* ./grobid-superconductors/ -COPY gradle.properties ./grobid-superconductors/ +RUN mkdir -p grobid-home/models \ + && mkdir -p grobid-superconductors_source/resources/config grobid-superconductors_source/resources/models grobid-superconductors_source/gradle grobid-superconductors_source/localLibs grobid-superconductors_source/resources/web grobid-superconductors_source/src -# Adjust config -RUN sed -i '/#Docker-ignore-log-start/,/#Docker-ignore-log-end/d' ./grobid-superconductors/resources/config/config-docker.yml +COPY resources/models/ ./grobid-superconductors_source/resources/models/ +COPY resources/config/ ./grobid-superconductors_source/resources/config/ +COPY gradle/ ./grobid-superconductors_source/gradle/ +COPY src/ ./grobid-superconductors_source/src/ +COPY localLibs/ ./grobid-superconductors_source/localLibs/ +COPY ["gradlew*", "build.gradle", "settings.gradle", "gradle.properties", "./grobid-superconductors_source/"] +COPY .git/ ./grobid-superconductors_source/.git/ # Preparing models -RUN rm -rf /opt/grobid-source/grobid-home/models/* +WORKDIR /opt/grobid-source/grobid-superconductors_source +RUN git remote prune origin && git repack && git prune-packed && git reflog expire --expire=1.day.ago && git gc --aggressive \ + && ./gradlew downloadTransformers --no-daemon --info --stacktrace \ + && rm -f /opt/grobid-source/grobid-home/models/*.zip \ + && rm -rf /opt/grobid-source/grobid-home/models/*.-with_ELMo \ + && rm -rf /opt/grobid-source/grobid-home/models/entityLinker* \ + && ./gradlew clean assemble -x shadowJar --no-daemon --stacktrace --info \ + && unzip -o build/distributions/grobid-superconductors-*.zip -d ../grobid-superconductors_distribution \ + && mv ../grobid-superconductors_distribution/grobid-superconductors-* ../grobid-superconductors \ + && rm -rf ../grobid-superconductors_distribution \ + && rm -rf /opt/grobid-source/grobid-superconductors_source/.git -WORKDIR /opt/grobid-source/grobid-quantities -RUN ./gradlew copyModels --no-daemon --info --stacktrace -WORKDIR /opt/grobid-source/grobid-superconductors -RUN ./gradlew clean assemble --no-daemon --info --stacktrace -RUN ./gradlew downloadTransformers --no-daemon --info --stacktrace && rm -f /opt/grobid-source/grobid-home/models/*.zip +# Preparing distribution +WORKDIR /opt/grobid-source/grobid-superconductors_source WORKDIR /opt @@ -74,21 +65,22 @@ WORKDIR /opt # build runtime image # ------------------- -FROM grobid/grobid:0.7.2 as runtime +FROM lfoppiano/grobid-quantities:0.8.0 as runtime # setting locale is likely useless but to be sure ENV LANG C.UTF-8 -# Install SO dependencies -RUN apt-get update && \ - apt-get -y --no-install-recommends install git wget - WORKDIR /opt/grobid -RUN mkdir -p /opt/grobid/grobid-superconductors +RUN rm -rf /opt/grobid/grobid-quantities \ + && rm /opt/grobid/resources \ + && mkdir -p /opt/grobid/grobid-superconductors \ + && rm -rf /opt/grobid/grobid-home/models/*.-with_ELMo \ + && rm -rf /opt/grobid/grobid-service + COPY --from=builder /opt/grobid-source/grobid-home/models ./grobid-home/models -COPY --from=builder /opt/grobid-source/grobid-superconductors/build/libs/* ./grobid-superconductors/ -COPY --from=builder /opt/grobid-source/grobid-superconductors/resources/config/config.yml ./grobid-superconductors/ +COPY --from=builder /opt/grobid-source/grobid-superconductors ./grobid-superconductors/ +COPY --from=builder /opt/grobid-source/grobid-superconductors_source/resources/config/config-docker.yml ./grobid-superconductors/resources/config/config.yml VOLUME ["/opt/grobid/grobid-home/tmp"] @@ -96,13 +88,6 @@ RUN pip install -U git+https://github.com/kermitt2/delft.git WORKDIR /opt/grobid -#RUN sed -i 's/pythonVirtualEnv:.*/pythonVirtualEnv: \/opt\/grobid\/venv/g' grobid-superconductors/config.yml -RUN sed -i 's/pythonVirtualEnv:.*/pythonVirtualEnv: /g' grobid-superconductors/config.yml -RUN sed -i 's/grobidHome:.*/grobidHome: grobid-home/g' grobid-superconductors/config.yml -RUN sed -i 's/chemDataExtractorUrl:.*/chemDataExtractorUrl: ${CDE_URL:- http:\/\/cde.local:8080}/g' grobid-superconductors/config.yml -RUN sed -i 's/linkingModuleUrl:.*/linkingModuleUrl: ${LINKING_MODULE_URL:- http:\/\/linking_module.local:8080}/g' grobid-superconductors/config.yml -RUN sed -i 's/classResolverUrl:.*/classResolverUrl: ${LINKING_MODULE_URL:- http:\/\/linking_module.local:8080}/g' grobid-superconductors/config.yml - ## Select transformers model ARG TRANSFORMERS_MODEL @@ -113,18 +98,24 @@ RUN if [[ -z "$TRANSFORMERS_MODEL" ]] ; then echo "Using Scibert as default tran # tar -xzf /tmp/jprofiler_linux_12_0_2.tar.gz -C /usr/local &&\ # rm /tmp/jprofiler_linux_12_0_2.tar.gz -EXPOSE 8072 8073 - +WORKDIR /opt/grobid ARG GROBID_VERSION ENV GROBID_VERSION=${GROBID_VERSION:-latest} +ENV GROBID_SUPERCONDUCTORS_OPTS "-Djava.library.path=/opt/grobid/grobid-home/lib/lin-64:/usr/local/lib/python3.8/dist-packages/jep --add-opens java.base/java.lang=ALL-UNNAMED" +ENV LINKING_MODULE_URL "http://linking_module.local:8080" +ENV CDE_URL "http://cde.local:8080" + +EXPOSE 8072 8073 -RUN if [ ! -f "grobid-superconductors/grobid-superconductors-${GROBID_VERSION}-onejar.jar" ]; then mv grobid-superconductors/grobid-superconductors-*-onejar.jar grobid-superconductors/grobid-superconductors-${GROBID_VERSION}-onejar.jar; fi +# RUN if [ ! -f "grobid-superconductors/grobid-superconductors-${GROBID_VERSION}-onejar.jar" ]; then mv grobid-superconductors/grobid-superconductors-*-onejar.jar grobid-superconductors/grobid-superconductors-${GROBID_VERSION}-onejar.jar; fi #RUN if [ "${!GROBID_VERSION}" = "unknown" ] ; then GROBID_VERSION=`ls grobid-superconductors/grobid-superconductors-*onejar.jar | grep -oE '[0-9]\.[0-9]\.[0-9](-SNAPSHOT)?' | head -n 1`; fi #CMD ["java", "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005", "-jar", "grobid-superconductors/grobid-superconductors-0.5.2-SNAPSHOT-onejar.jar", "server", "grobid-superconductors/config.yml"] #CMD ["java", "-agentpath:/usr/local/jprofiler12.0.2/bin/linux-x64/libjprofilerti.so=port=8849", "-jar", "grobid-superconductors/grobid-superconductors-0.2.1-SNAPSHOT-onejar.jar", "server", "grobid-superconductors/config.yml"] -CMD ["sh", "-c", "java -jar grobid-superconductors/grobid-superconductors-${GROBID_VERSION}-onejar.jar server grobid-superconductors/config.yml"] +#CMD ["sh", "-c", "java -jar grobid-superconductors/grobid-superconductors-${GROBID_VERSION}-onejar.jar server grobid-superconductors/config.yml"] + +CMD ["./grobid-superconductors/bin/grobid-superconductors", "server", "grobid-superconductors/resources/config/config.yml"] LABEL \ diff --git a/README.md b/README.md index d36f4e08..bfdb8346 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,11 @@ -# Grobid superconductors module +# Grobid superconductors [![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) [![Docker Hub](https://img.shields.io/docker/pulls/lfoppiano/grobid-superconductors.svg)](https://hub.docker.com/r/lfoppiano/grobid-superconductors/ "Docker Pulls") [![Coverage Status](https://coveralls.io/repos/github/lfoppiano/grobid-superconductors/badge.svg?branch=master)](https://coveralls.io/github/lfoppiano/grobid-superconductors?branch=master) -[![CircleCI](https://circleci.com/gh/lfoppiano/grobid-superconductors.svg?style=svg)](https://circleci.com/gh/lfoppiano/grobid-superconductors) -[![Build unstable](https://github.com/lfoppiano/grobid-superconductors/actions/workflows/ci-build-unstable.yml/badge.svg)](https://github.com/lfoppiano/grobid-superconductors/actions/workflows/ci-build-unstable.yml) - -__Work in progress.__ +[![Demo grobid-superconductors](https://img.shields.io/website-up-down-green-red/https/lfoppiano-grobid-superconductors.hf.space.svg)](https://lfoppiano-grobid-superconductors.hf.space) +[![Build](https://github.com/lfoppiano/grobid-superconductors/actions/workflows/ci-build-unstable.yml/badge.svg)](https://github.com/lfoppiano/grobid-superconductors/actions/workflows/ci-build-unstable.yml) +[![Open in Spaces](https://huggingface.co/datasets/huggingface/badges/raw/main/open-in-hf-spaces-sm.svg)](https://lfoppiano-grobid-superconductors.hf.space/) * [Introduction](#introduction) * [Getting started](#getting-started) @@ -419,6 +418,8 @@ Our warmest thanks to [Patrice Lopez](https://github.com/kermitt2) from [Science This project has been developed at the [National Institute for Materials Science](http://www.nims.go.jp), in [Tsukuba](https://en.wikipedia.org/wiki/Tsukuba,_Ibaraki), Japan. +Developed using ![JetBrains Logo (Main) logo](https://resources.jetbrains.com/storage/products/company/brand/logos/jb_beam.svg) + ## License GROBID and grobid-superconductors are distributed under [Apache 2.0 license](http://www.apache.org/licenses/LICENSE-2.0). @@ -432,56 +433,56 @@ We described the framework around the system in the following articles (the late - [Automatic Extraction of Materials and Properties from Superconductors Scientific Literature ](https://hal.inria.fr/hal-03776658) ``` - @article{doi:10.1080/27660400.2022.2153633, - author = {Luca Foppiano and Pedro Baptista Castro and Pedro Ortiz Suarez and Kensei Terashima and Yoshihiko Takano and Masashi Ishii}, - title = {Automatic extraction of materials and properties from superconductors scientific literature}, - journal = {Science and Technology of Advanced Materials: Methods}, - volume = {3}, - number = {1}, - pages = {2153633}, - year = {2023}, - publisher = {Taylor & Francis}, - doi = {10.1080/27660400.2022.2153633}, - URL = { - https://doi.org/10.1080/27660400.2022.2153633 - }, - eprint = { - https://doi.org/10.1080/27660400.2022.2153633 + @article{doi:10.1080/27660400.2022.2153633, + author = {Luca Foppiano and Pedro Baptista Castro and Pedro Ortiz Suarez and Kensei Terashima and Yoshihiko Takano and Masashi Ishii}, + title = {Automatic extraction of materials and properties from superconductors scientific literature}, + journal = {Science and Technology of Advanced Materials: Methods}, + volume = {3}, + number = {1}, + pages = {2153633}, + year = {2023}, + publisher = {Taylor & Francis}, + doi = {10.1080/27660400.2022.2153633}, + URL = { + https://doi.org/10.1080/27660400.2022.2153633 + }, + eprint = { + https://doi.org/10.1080/27660400.2022.2153633 + } } - } ``` - [SuperMat](http://github.com/lfoppiano/Supermat): construction of a linked annotated dataset from superconductors-related publications ``` - @article{doi:10.1080/27660400.2021.1918396, - author = {Luca Foppiano and Sae Dieb and Akira Suzuki and Pedro Baptista de Castro and Suguru Iwasaki and Azusa Uzuki and Miren Garbine Esparza Echevarria and Yan Meng and Kensei Terashima and Laurent Romary and Yoshihiko Takano and Masashi Ishii}, - title = {SuperMat: construction of a linked annotated dataset from superconductors-related publications}, - journal = {Science and Technology of Advanced Materials: Methods}, - volume = {1}, - number = {1}, - pages = {34-44}, - year = {2021}, - publisher = {Taylor & Francis}, - doi = {10.1080/27660400.2021.1918396}, - URL = { https://doi.org/10.1080/27660400.2021.1918396 }, - eprint = { https://doi.org/10.1080/27660400.2021.1918396 } - } + @article{doi:10.1080/27660400.2021.1918396, + author = {Luca Foppiano and Sae Dieb and Akira Suzuki and Pedro Baptista de Castro and Suguru Iwasaki and Azusa Uzuki and Miren Garbine Esparza Echevarria and Yan Meng and Kensei Terashima and Laurent Romary and Yoshihiko Takano and Masashi Ishii}, + title = {SuperMat: construction of a linked annotated dataset from superconductors-related publications}, + journal = {Science and Technology of Advanced Materials: Methods}, + volume = {1}, + number = {1}, + pages = {34-44}, + year = {2021}, + publisher = {Taylor & Francis}, + doi = {10.1080/27660400.2021.1918396}, + URL = { https://doi.org/10.1080/27660400.2021.1918396 }, + eprint = { https://doi.org/10.1080/27660400.2021.1918396 } + } ``` - "Proposal for Automatic Extraction of Superconductors properties from scientific literature": [PDF](http://pubman.nims.go.jp/pubman/faces/viewItemOverviewPage.jsp?itemId=escidoc:1890245:3) ``` - @inproceedings{foppiano2019proposal, - address = {Tsukuba}, - title = {Proposal for {Automatic} {Extraction} {Framework} of {Superconductors} {Related} {Information} from {Scientific} {Literature}}, - volume = {119}, - copyright = {All rights reserved}, - abstract = {The automatic collection of materials information from research papers using Natural Language Processing (NLP) is highly required for rapid materials development using big data, namely materials informatics (MI). The difficulty of this automatic collection is mainly caused by the variety of expressions in the papers, a robust system with tolerance to such variety is required to be developed. In this paper, we report an ongoing interdisciplinary work to construct a system for automatic collection of superconductor-related information from scientific literature using text mining techniques. We focused on the identification of superconducting material names and their critical temperature (Tc) key property. We discuss the construction of a prototype for extraction and linking using machine learning (ML) techniques for the physical information collection. From the evaluation using 500 sample documents, we define a baseline and a direction for future improvements.}, - language = {eng}, - booktitle = {Letters and {Technology} {News}, vol. 119, no. 66, {SC}2019-1 (no.66)}, - author = {Foppiano, Luca and Thaer, M. Dieb and Suzuki, Akira and Ishii, Masashi}, - month = may, - year = {2019}, - note = {ISSN: 2432-6380}, - pages = {1--5} - } + @inproceedings{foppiano2019proposal, + address = {Tsukuba}, + title = {Proposal for {Automatic} {Extraction} {Framework} of {Superconductors} {Related} {Information} from {Scientific} {Literature}}, + volume = {119}, + copyright = {All rights reserved}, + abstract = {The automatic collection of materials information from research papers using Natural Language Processing (NLP) is highly required for rapid materials development using big data, namely materials informatics (MI). The difficulty of this automatic collection is mainly caused by the variety of expressions in the papers, a robust system with tolerance to such variety is required to be developed. In this paper, we report an ongoing interdisciplinary work to construct a system for automatic collection of superconductor-related information from scientific literature using text mining techniques. We focused on the identification of superconducting material names and their critical temperature (Tc) key property. We discuss the construction of a prototype for extraction and linking using machine learning (ML) techniques for the physical information collection. From the evaluation using 500 sample documents, we define a baseline and a direction for future improvements.}, + language = {eng}, + booktitle = {Letters and {Technology} {News}, vol. 119, no. 66, {SC}2019-1 (no.66)}, + author = {Foppiano, Luca and Thaer, M. Dieb and Suzuki, Akira and Ishii, Masashi}, + month = may, + year = {2019}, + note = {ISSN: 2432-6380}, + pages = {1--5} + } ``` diff --git a/app.json b/app.json deleted file mode 100644 index 5f65af9a..00000000 --- a/app.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "grobid-superconductors", - "description": "Open source machine-learning based for extracting superconductors related information and properties.", - "keywords": ["Text mining", "Machine Learning", "Materials informatics", "Superconductors"], - "website": "https://grobid-superconductors.herokuapps.com/", - "repository": "https://github.com/lfoppiano/grobid-superconductors", - "success_url": "/", - "env": { - "ADMIN_CONTACT_EMAIL": { - "description": "The contact email address for the admin account." - } - }, - "stack": "container" -} \ No newline at end of file diff --git a/build.gradle b/build.gradle index f70fed0e..2e60fd45 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,5 @@ +import org.apache.tools.ant.taskdefs.condition.Os + buildscript { repositories { mavenLocal() @@ -19,9 +21,10 @@ plugins { id 'maven-publish' id 'com.github.johnrengelman.shadow' version '7.0.0' id "de.undercouch.download" version "4.1.1" - id 'com.github.kt3k.coveralls' version '2.12.0' + id 'com.github.kt3k.coveralls' version '2.12.2' id 'com.palantir.git-version' version '0.12.3' id 'net.researchgate.release' version '3.0.2' + id "org.jetbrains.kotlin.jvm" version "1.8.21" } @@ -44,9 +47,12 @@ description = """Grobid module for extraction of superconductor information from dependencies { //Tests - testImplementation(platform('org.junit:junit-bom:5.8.1')) + testImplementation(platform('org.junit:junit-bom:5.10.2')) + testRuntimeOnly("org.junit.platform:junit-platform-launcher") { + because("Only needed to run tests in a version of IntelliJ IDEA that bundles older versions") + } + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") testImplementation('org.junit.jupiter:junit-jupiter') - testImplementation("junit:junit:4.13.2") testRuntimeOnly("org.junit.vintage:junit-vintage-engine") { because 'allows JUnit 3 and JUnit 4 tests to run' } @@ -54,16 +60,19 @@ dependencies { testRuntimeOnly("org.junit.platform:junit-platform-launcher") { because 'allows tests to run from IDEs that bundle older version of launcher' } - testImplementation 'org.easymock:easymock:4.3' + testImplementation 'org.easymock:easymock:5.2.0' + testImplementation 'org.hamcrest:hamcrest-all:1.3' testImplementation 'org.hamcrest:hamcrest-library:2.2' testImplementation 'org.powermock:powermock-module-junit4:2.0.9' testImplementation 'org.powermock:powermock-api-easymock:2.0.9' + testImplementation 'org.jetbrains.kotlin:kotlin-test' + testImplementation "io.mockk:mockk:1.13.9" //GROBID and grobid-quantities - implementation 'com.github.kermitt2:grobid-quantities:0.7.2' - implementation 'org.grobid:grobid-core:0.7.2' - implementation 'org.grobid:grobid-trainer:0.7.2' - implementation 'org.grobid:grobid-service:0.7.2' + implementation 'org.grobid:grobid-quantities:0.8.0' + implementation 'org.grobid:grobid-core:0.8.0' + implementation 'org.grobid:grobid-trainer:0.8.0' + implementation 'org.grobid:grobid-service:0.8.0' implementation "xerces:xercesImpl:2.12.0" implementation "net.arnx:jsonic:1.3.10" implementation "net.sf.saxon:Saxon-HE:9.6.0-9" @@ -83,25 +92,36 @@ dependencies { implementation 'org.apache.commons:commons-text:1.10.0' //Dropwizard - implementation "io.dropwizard:dropwizard-core:1.3.29" - implementation 'io.dropwizard:dropwizard-jersey:1.3.29' - implementation "io.dropwizard:dropwizard-assets:1.3.29" - implementation "com.hubspot.dropwizard:dropwizard-guicier:1.3.5.2" - implementation "io.dropwizard:dropwizard-testing:1.3.29" - implementation "io.dropwizard:dropwizard-forms:1.3.29" - implementation "io.dropwizard:dropwizard-client:1.3.29" - implementation "io.dropwizard:dropwizard-auth:1.3.29" - implementation "io.dropwizard.metrics:metrics-core:4.0.0" - implementation "io.dropwizard.metrics:metrics-servlets:4.0.0" + implementation 'ru.vyarus:dropwizard-guicey:7.0.0' + + implementation 'io.dropwizard:dropwizard-bom:4.0.0' + implementation 'io.dropwizard:dropwizard-core:4.0.0' + implementation 'io.dropwizard:dropwizard-assets:4.0.0' + implementation 'io.dropwizard:dropwizard-testing:4.0.0' + implementation 'io.dropwizard:dropwizard-forms:4.0.0' + implementation 'io.dropwizard:dropwizard-client:4.0.0' + implementation 'io.dropwizard:dropwizard-auth:4.0.0' + implementation 'io.dropwizard.metrics:metrics-core:4.2.22' + implementation 'io.dropwizard.metrics:metrics-servlets:4.2.22' //Misc - implementation 'com.google.guava:guava:31.1-jre' + implementation 'com.google.guava:guava:30.1.1-jre' implementation 'org.apache.httpcomponents:httpmime:4.5.13' implementation 'javax.activation:activation:1.1.1' implementation 'javax.xml.bind:jaxb-api:2.4.0-b180830.0359' implementation 'com.sun.xml.bind:jaxb-core:4.0.1' implementation 'com.sun.xml.bind:jaxb-impl:4.0.1' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-guava:2.13.4' + + //Grobid-quantities + implementation 'tech.units:indriya:2.1.3' + implementation group: 'si.uom', name: 'si-units', version: '2.1' + implementation group: 'si.uom', name: 'si-quantity', version: '2.1' + implementation group: 'systems.uom', name: 'systems-quantity', version: '2.1' + implementation group: 'systems.uom', name: 'systems-common', version: '2.1' + implementation group: 'systems.uom', name: 'systems-unicode', version: '2.1' + implementation group: 'systems.uom', name: 'systems-ucum', version: '2.1' + implementation group: 'systems.uom', name: 'systems-unicode', version: '2.1' implementation 'org.dkpro.statistics:dkpro-statistics:2.2.1' implementation 'org.dkpro.statistics:dkpro-statistics-agreement:2.2.1' @@ -125,9 +145,6 @@ configurations.all { } /** Other configuration **/ -sourceCompatibility = 1.8 -targetCompatibility = 1.8 - tasks.withType(JavaCompile) { options.encoding = 'UTF-8' // options.compilerArgs << "-Xlint:deprecation" @@ -137,9 +154,40 @@ tasks.withType(JavaCompile) { sourceSets.main.resources { srcDirs = ["src/main/resources", "resources/config"] } +sourceSets.test.resources { + srcDirs = ["src/test/resources"] +} test { exclude '**/**IntegrationTest**' + useJUnitPlatform() + maxHeapSize = "1024m" + + def libraries = "" + if (Os.isFamily(Os.FAMILY_MAC)) { + if (Os.isArch("aarch64")) { + libraries = "${file("./grobid-home/lib/mac_arm-64").absolutePath}" + } else { + libraries = "${file("./grobid-home/lib/mac-64").absolutePath}" + } + } else if (Os.isFamily(Os.FAMILY_UNIX)) { + def jepDir = rootProject.rootDir.getAbsolutePath() + "/grobid-home/lib/lin-64/jep" + libraries = jepDir + jepDir = rootProject.rootDir.getAbsolutePath() + "/grobid-home/lib/lin-64" + libraries += ":" + jepDir + } else { + throw new RuntimeException("Unsupported platform!") + } + + if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) { + jvmArgs "--add-opens", "java.base/java.util.stream=ALL-UNNAMED", + "--add-opens", "java.base/java.io=ALL-UNNAMED", + "--add-opens", "java.base/java.lang=ALL-UNNAMED", + "--add-opens", "java.base/java.util.regex=ALL-UNNAMED", + "--add-opens", "java.base/java.math=ALL-UNNAMED", + "--add-opens", "java.base/java.text=ALL-UNNAMED" + } + systemProperty "java.library.path", "${System.getProperty('java.library.path')}:" + libraries } tasks.run.workingDir = rootProject.rootDir @@ -148,6 +196,60 @@ mainClassName = 'org.grobid.service.GrobidSuperconductorsApplication' run { args = ['server', 'resources/config/config.yml'] + + def libraries = "" + if (Os.isFamily(Os.FAMILY_MAC)) { + if (Os.isArch("aarch64")) { + libraries = "${file("../grobid-home/lib/mac_arm-64").absolutePath}" + } else { + libraries = "${file("../grobid-home/lib/mac-64").absolutePath}" + } + } else if (Os.isFamily(Os.FAMILY_UNIX)) { + libraries = "${file("../grobid-home/lib/lin-64/jep").absolutePath}:" + + "${file("../grobid-home/lib/lin-64").absolutePath}:" + } else { + throw new RuntimeException("Unsupported platform!") + } + + if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) { + jvmArgs "--add-opens", "java.base/java.lang=ALL-UNNAMED" + } + workingDir = rootProject.rootDir + def javaLibraryPath = "${System.getProperty('java.library.path')}:" + libraries + + systemProperty "java.library.path", javaLibraryPath +} + +task integration(type: Test) { + include '**' + + maxHeapSize = "1024m" + + def libraries = "" + if (Os.isFamily(Os.FAMILY_MAC)) { + if (Os.isArch("aarch64")) { + libraries = "${file("./grobid-home/lib/mac_arm-64").absolutePath}" + } else { + libraries = "${file("./grobid-home/lib/mac-64").absolutePath}" + } + } else if (Os.isFamily(Os.FAMILY_UNIX)) { + def jepDir = rootProject.rootDir.getAbsolutePath() + "/grobid-home/lib/lin-64/jep" + libraries = jepDir + jepDir = rootProject.rootDir.getAbsolutePath() + "/grobid-home/lib/lin-64" + libraries += ":" + jepDir + } else { + throw new RuntimeException("Unsupported platform!") + } + + if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) { + jvmArgs "--add-opens", "java.base/java.util.stream=ALL-UNNAMED", + "--add-opens", "java.base/java.io=ALL-UNNAMED", + "--add-opens", "java.base/java.lang=ALL-UNNAMED", + "--add-opens", "java.base/java.util.regex=ALL-UNNAMED", + "--add-opens", "java.base/java.math=ALL-UNNAMED", + "--add-opens", "java.base/java.text=ALL-UNNAMED" + } + systemProperty "java.library.path", "${System.getProperty('java.library.path')}:" + libraries } @@ -201,18 +303,14 @@ shadowJar { } jar { - enabled false + enabled true } artifacts { archives shadowJar -// archives jar } -//distZip.archiveClassifier = 'application' -//distTar.archiveClassifier = 'application' - -distZip.enabled = false +distZip.enabled = true distTar.enabled = false shadowDistZip.enabled = false shadowDistTar.enabled = false @@ -230,6 +328,9 @@ publishing { def conf = new org.yaml.snakeyaml.Yaml().load( new File("resources/config/config.yml").newInputStream() ) def grobidHome = conf.grobidHome.replace("\$", "").replace('{', "").replace("GROBID_HOME:- ", "").replace("}", "") +if (grobidHome.startsWith("../")) { + grobidHome = "${rootProject.rootDir}/${grobidHome}" +} /** Model management **/ @@ -244,29 +345,44 @@ task copyModels(type: Copy) { include "**/preprocessor.json" exclude "**/features-engineering/**" exclude "**/result-logs/**" - into "${rootDir}/${grobidHome}/models/" + into "${grobidHome}/models/" doLast { print "Copy models under grobid-home: ${grobidHome}" - ant.gunzip(src: "${rootDir}/${grobidHome}/models/superconductors/model.wapiti.gz", dest: "${rootDir}/${grobidHome}/models/superconductors/") + ant.gunzip(src: "${grobidHome}/models/superconductors/model.wapiti.gz", dest: "${grobidHome}/models/superconductors/") } } -task downloadTransformers(dependsOn: copyModels) { +task downloadTransformersQuantities(dependsOn: copyModels) { doLast { download { - src "https://kdrive.infomaniak.com/2/app/104844/share/0f902484-a529-4562-96d7-364959f364dd/files/150668/download" - dest "${rootDir}/${grobidHome}/models/superconductors-transformers.zip" + src "https://transformers-data.s3.eu-central-1.amazonaws.com/quantities-transformers-240226.zip" + dest "${grobidHome}/models/quantities-transformers.zip" overwrite false print "Download bulky transformers files under grobid-home: ${grobidHome}" } - ant.unzip(src: "${rootDir}/${grobidHome}/models/superconductors-transformers.zip", dest: "${rootDir}/${grobidHome}/models/") + ant.unzip(src: "${grobidHome}/models/quantities-transformers.zip", dest: "${grobidHome}/models/") } } +task downloadTransformers(dependsOn: copyModels) { + doLast { + download { + src "https://transformers-data.s3.eu-central-1.amazonaws.com/superconductors-transformers-230222.zip" + dest "${grobidHome}/models/superconductors-transformers.zip" + overwrite false + print "Download bulky transformers files under grobid-home: ${grobidHome}" + } + ant.unzip(src: "${grobidHome}/models/superconductors-transformers.zip", dest: "${grobidHome}/models/") + } +} wrapper { - gradleVersion "7.1.1" + gradleVersion "7.2" +} + +test { + useJUnitPlatform() } jacocoTestReport { @@ -289,5 +405,3 @@ release { requireBranch.set('master') } } - - diff --git a/docker-compose.yml b/docker-compose.yml index eb22ca36..1a08c821 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - "cde.local" linking_module: - image: lfoppiano/grobid-superconductors-tools:2.1 + image: lfoppiano/material-parsers:3.0.1 restart: always ports: - 8090:8080 @@ -27,7 +27,7 @@ services: superconductors: - image: lfoppiano/grobid-superconductors:0.5.0 + image: lfoppiano/grobid-superconductors:0.5.3 volumes: - ./resources/config/config-docker.yml:/opt/grobid/grobid-superconductors/config.yml:ro # - /Users/lfoppiano/development/projects/grobid/grobid-superconductors/resources/config/grobid.yaml:/opt/grobid/grobid-home/config/grobid.yaml:ro diff --git a/gradle.properties b/gradle.properties index bfec3755..5aef8ae5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,8 @@ -version=0.5.3-SNAPSHOT +version=0.5.5-SNAPSHOT org.gradle.daemon=false org.gradle.caching = true org.gradle.parallel = true org.gradle.vfs.watch = true + diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e708b1c0..7454180f 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 05679dc3..ffed3a25 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 4f906e0c..1b6c7873 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,101 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -106,80 +140,95 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/localLibs/grobid-core-0.7.2.jar b/localLibs/grobid-core-0.8.0.jar similarity index 93% rename from localLibs/grobid-core-0.7.2.jar rename to localLibs/grobid-core-0.8.0.jar index d923517c..b5edf72f 100644 Binary files a/localLibs/grobid-core-0.7.2.jar and b/localLibs/grobid-core-0.8.0.jar differ diff --git a/localLibs/grobid-quantities-0.8.0.jar b/localLibs/grobid-quantities-0.8.0.jar new file mode 100644 index 00000000..57abcff3 Binary files /dev/null and b/localLibs/grobid-quantities-0.8.0.jar differ diff --git a/localLibs/grobid-service-0.7.2.jar b/localLibs/grobid-service-0.8.0.jar similarity index 94% rename from localLibs/grobid-service-0.7.2.jar rename to localLibs/grobid-service-0.8.0.jar index b0e802d8..f2f86e34 100644 Binary files a/localLibs/grobid-service-0.7.2.jar and b/localLibs/grobid-service-0.8.0.jar differ diff --git a/localLibs/grobid-trainer-0.7.2.jar b/localLibs/grobid-trainer-0.8.0.jar similarity index 96% rename from localLibs/grobid-trainer-0.7.2.jar rename to localLibs/grobid-trainer-0.8.0.jar index 797224df..780c0f2a 100644 Binary files a/localLibs/grobid-trainer-0.7.2.jar and b/localLibs/grobid-trainer-0.8.0.jar differ diff --git a/resources/config/config-docker.yml b/resources/config/config-docker.yml index 37a5ee27..336206dc 100755 --- a/resources/config/config-docker.yml +++ b/resources/config/config-docker.yml @@ -27,12 +27,12 @@ consolidation: # empty value will disable the data consolidation service: "glutton" glutton: - url: "http://falcon.nims.go.jp/glutton" + url: "http://falcon.nims.go.jp/glutton/lookup" models: - name: "superconductors" - engine: "wapiti" - # engine: "delft" +# engine: "wapiti" + engine: "delft" wapiti: # wapiti training parameters, they will be used at training time only epsilon: 0.00001 @@ -41,20 +41,22 @@ models: delft: # deep learning parameters # architecture: "BidLSTM_CRF" - architecture: "scibert" - embeddings_name: "glove-840B" + architecture: "BERT_CRF" +# embeddings_name: "glove-840B" runtime: # parameters used at runtime/prediction - max_sequence_length: 3000 + max_sequence_length: 512 batch_size: 4 - name: "material" - engine: "wapiti" + engine: "delft" wapiti: # wapiti training parameters, they will be used at training time only epsilon: 0.00001 window: 30 nbMaxIterations: 2000 + delft: + architecture: "BERT_CRF" - name: "entityLinker-material-tcValue" engine: "wapiti" @@ -81,42 +83,58 @@ models: nbMaxIterations: 2000 - name: "quantities" - engine: "wapiti" + engine: "delft" +# engine: "wapiti" wapiti: # wapiti training parameters, they will be used at training time only epsilon: 0.00001 window: 30 nbMaxIterations: 2000 + delft: +# architecture: "BidLSTM_CRF_FEATURES" + architecture: "BERT_CRF" + useELMo: false + embeddings_name: "glove-840B" - name: "units" - engine: "wapiti" + engine: "delft" +# engine: "wapiti" wapiti: # wapiti training parameters, they will be used at training time only epsilon: 0.00001 window: 30 nbMaxIterations: 2000 + delft: + architecture: "BidLSTM_CRF_FEATURES" + useELMo: false + embeddings_name: "glove-840B" - name: "values" - engine: "wapiti" + engine: "delft" +# engine: "wapiti" wapiti: # wapiti training parameters, they will be used at training time only epsilon: 0.00001 window: 30 nbMaxIterations: 2000 + delft: + architecture: "BidLSTM_CRF_FEATURES" + useELMo: false + embeddings_name: "glove-840B" server: type: custom - idleTimeout: 120 seconds applicationConnectors: - type: http port: 8072 + idleTimeout: 120 seconds adminConnectors: - type: http port: 8073 registerDefaultExceptionMappers: false logging: - level: DEBUG + level: WARN # Logger-specific levels. loggers: @@ -128,6 +146,4 @@ logging: appenders: - type: console threshold: INFO - -version: 2 -timeZone: UTC + timeZone: UTC \ No newline at end of file diff --git a/resources/config/config.yml b/resources/config/config.yml index fdcd4a0c..c8520fe7 100755 --- a/resources/config/config.yml +++ b/resources/config/config.yml @@ -31,8 +31,8 @@ consolidation: models: - name: "superconductors" - engine: "wapiti" -# engine: "delft" +# engine: "wapiti" + engine: "delft" wapiti: # wapiti training parameters, they will be used at training time only epsilon: 0.00001 @@ -49,12 +49,14 @@ models: batch_size: 4 - name: "material" - engine: "wapiti" + engine: "delft" wapiti: # wapiti training parameters, they will be used at training time only epsilon: 0.00001 window: 30 nbMaxIterations: 2000 + delft: + architecture: "BERT_CRF" - name: "entityLinker-material-tcValue" engine: "wapiti" @@ -121,10 +123,10 @@ models: server: type: custom - idleTimeout: 120 seconds applicationConnectors: - type: http port: 8072 + idleTimeout: 120 seconds adminConnectors: - type: http port: 8073 @@ -143,6 +145,7 @@ logging: appenders: - type: console threshold: INFO + timeZone: UTC # The following line is used to remove the logger in the docker image, please don't modify it #Docker-ignore-log-start - type: file @@ -154,6 +157,3 @@ logging: timeZone: UTC maxFileSize: 50MB #Docker-ignore-log-end - -version: 2 -timeZone: UTC diff --git a/resources/models/grobid-superconductors-positive_sampling-all-BidLSTM_CRF_FEATURES-with_ELMo/config.json b/resources/models/grobid-superconductors-positive_sampling-all-BidLSTM_CRF_FEATURES-with_ELMo/config.json new file mode 100644 index 00000000..5e81a154 --- /dev/null +++ b/resources/models/grobid-superconductors-positive_sampling-all-BidLSTM_CRF_FEATURES-with_ELMo/config.json @@ -0,0 +1,91 @@ +{ + "model_name": "grobid-superconductors-positive_sampling-all-BidLSTM_CRF_FEATURES-with_ELMo", + "architecture": "BidLSTM_CRF_FEATURES", + "embeddings_name": "glove-840B", + "char_vocab_size": 227, + "case_vocab_size": 8, + "char_embedding_size": 25, + "num_char_lstm_units": 25, + "max_char_length": 30, + "features_vocabulary_size": 12, + "features_indices": [ + 9, + 10, + 11, + 12, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "features_embedding_size": 4, + "features_lstm_units": 4, + "max_sequence_length": 3000, + "word_embedding_size": 1324, + "num_word_lstm_units": 100, + "case_embedding_size": 5, + "dropout": 0.5, + "recurrent_dropout": 0.5, + "use_crf": true, + "use_chain_crf": false, + "fold_number": 1, + "batch_size": 20, + "transformer_name": null, + "use_ELMo": true, + "features_map_to_index": { + "9": { + "ALLCAPS": 1, + "INITCAP": 2, + "NOCAPS": 3 + }, + "10": { + "ALLDIGIT": 13, + "ALLDIGITS": 14, + "CONTAINDIGIT": 15, + "NODIGIT": 16 + }, + "11": { + "0": 25, + "1": 26 + }, + "12": { + "COMMA": 37, + "DOT": 38, + "ENDBRACKET": 39, + "HYPHEN": 40, + "NOPUNCT": 41, + "OPENBRACKET": 42, + "PUNCT": 43, + "QUOTE": 44 + }, + "16": { + "DIFFERENTFONT": 49, + "SAMEFONT": 50, + "SAxMEFONT": 51 + }, + "17": { + "HIGHERFONT": 61, + "LOWERFONT": 62, + "SAMEFONTSIZE": 63 + }, + "18": { + "false": 73, + "true": 74 + }, + "19": { + "false": 85, + "true": 86 + }, + "20": { + "BASELINE": 97, + "SUBSCRIPT": 98, + "SUPERSCRIPT": 99 + }, + "21": { + "false": 109, + "true": 110 + } + } +} \ No newline at end of file diff --git a/resources/models/grobid-superconductors-positive_sampling-all-BidLSTM_CRF_FEATURES-with_ELMo/model_weights.hdf5 b/resources/models/grobid-superconductors-positive_sampling-all-BidLSTM_CRF_FEATURES-with_ELMo/model_weights.hdf5 new file mode 100644 index 00000000..40b8b002 Binary files /dev/null and b/resources/models/grobid-superconductors-positive_sampling-all-BidLSTM_CRF_FEATURES-with_ELMo/model_weights.hdf5 differ diff --git a/resources/models/grobid-superconductors-positive_sampling-all-BidLSTM_CRF_FEATURES-with_ELMo/preprocessor.json b/resources/models/grobid-superconductors-positive_sampling-all-BidLSTM_CRF_FEATURES-with_ELMo/preprocessor.json new file mode 100644 index 00000000..ee885822 --- /dev/null +++ b/resources/models/grobid-superconductors-positive_sampling-all-BidLSTM_CRF_FEATURES-with_ELMo/preprocessor.json @@ -0,0 +1,350 @@ +{ + "padding": true, + "return_lengths": true, + "return_word_embeddings": true, + "return_casing": false, + "return_features": true, + "return_chars": true, + "return_bert_embeddings": false, + "vocab_char": { + "": 0, + "": 1, + "!": 2, + "\"": 3, + "#": 4, + "$": 5, + "%": 6, + "&": 7, + "'": 8, + "(": 9, + ")": 10, + "*": 11, + "+": 12, + ",": 13, + "-": 14, + ".": 15, + "/": 16, + "0": 17, + "1": 18, + "2": 19, + "3": 20, + "4": 21, + "5": 22, + "6": 23, + "7": 24, + "8": 25, + "9": 26, + ":": 27, + ";": 28, + "<": 29, + "=": 30, + ">": 31, + "?": 32, + "@": 33, + "A": 34, + "B": 35, + "C": 36, + "D": 37, + "E": 38, + "F": 39, + "G": 40, + "H": 41, + "I": 42, + "J": 43, + "K": 44, + "L": 45, + "M": 46, + "N": 47, + "O": 48, + "P": 49, + "Q": 50, + "R": 51, + "S": 52, + "T": 53, + "U": 54, + "V": 55, + "W": 56, + "X": 57, + "Y": 58, + "Z": 59, + "[": 60, + "]": 61, + "_": 62, + "`": 63, + "a": 64, + "b": 65, + "c": 66, + "d": 67, + "e": 68, + "f": 69, + "g": 70, + "h": 71, + "i": 72, + "j": 73, + "k": 74, + "l": 75, + "m": 76, + "n": 77, + "o": 78, + "p": 79, + "q": 80, + "r": 81, + "s": 82, + "t": 83, + "u": 84, + "v": 85, + "w": 86, + "x": 87, + "y": 88, + "z": 89, + "{": 90, + "|": 91, + "}": 92, + "~": 93, + "\u00a9": 94, + "\u00ab": 95, + "\u00b0": 96, + "\u00b1": 97, + "\u00b4": 98, + "\u00b5": 99, + "\u00b7": 100, + "\u00ba": 101, + "\u00bb": 102, + "\u00bc": 103, + "\u00bd": 104, + "\u00c0": 105, + "\u00c2": 106, + "\u00c3": 107, + "\u00c5": 108, + "\u00c8": 109, + "\u00d0": 110, + "\u00d7": 111, + "\u00d8": 112, + "\u00d9": 113, + "\u00de": 114, + "\u00df": 115, + "\u00e1": 116, + "\u00e2": 117, + "\u00e3": 118, + "\u00e4": 119, + "\u00e8": 120, + "\u00e9": 121, + "\u00eb": 122, + "\u00ec": 123, + "\u00ee": 124, + "\u00f0": 125, + "\u00f1": 126, + "\u00f6": 127, + "\u00f8": 128, + "\u00fc": 129, + "\u00fe": 130, + "\u0109": 131, + "\u01eb": 132, + "\u01fa": 133, + "\u02da": 134, + "\u0351": 135, + "\u0352": 136, + "\u035e": 137, + "\u0393": 138, + "\u0394": 139, + "\u0398": 140, + "\u039b": 141, + "\u03a6": 142, + "\u03a8": 143, + "\u03a9": 144, + "\u03b1": 145, + "\u03b2": 146, + "\u03b3": 147, + "\u03b4": 148, + "\u03b5": 149, + "\u03b6": 150, + "\u03b7": 151, + "\u03b8": 152, + "\u03ba": 153, + "\u03bb": 154, + "\u03bc": 155, + "\u03bd": 156, + "\u03be": 157, + "\u03c0": 158, + "\u03c1": 159, + "\u03c3": 160, + "\u03c4": 161, + "\u03c6": 162, + "\u03c7": 163, + "\u03c9": 164, + "\u03f3": 165, + "\u2022": 166, + "\u2026": 167, + "\u2032": 168, + "\u2113": 169, + "\u2126": 170, + "\u2190": 171, + "\u2192": 172, + "\u2194": 173, + "\u21d4": 174, + "\u21e0": 175, + "\u2202": 176, + "\u2206": 177, + "\u2212": 178, + "\u221a": 179, + "\u221d": 180, + "\u221e": 181, + "\u222b": 182, + "\u223c": 183, + "\u2243": 184, + "\u2245": 185, + "\u2248": 186, + "\u2260": 187, + "\u2261": 188, + "\u2264": 189, + "\u2265": 190, + "\u226a": 191, + "\u226b": 192, + "\u2272": 193, + "\u22a5": 194, + "\u22c5": 195, + "\u22c6": 196, + "\u23a2": 197, + "\u2500": 198, + "\u25b3": 199, + "\uf02d": 200, + "\uf03c": 201, + "\uf03e": 202, + "\uf044": 203, + "\uf057": 204, + "\uf05e": 205, + "\uf061": 206, + "\uf062": 207, + "\uf063": 208, + "\uf064": 209, + "\uf067": 210, + "\uf06c": 211, + "\uf06d": 212, + "\uf070": 213, + "\uf072": 214, + "\uf073": 215, + "\uf077": 216, + "\uf0a0": 217, + "\uf0a3": 218, + "\uf0b3": 219, + "\uf0b4": 220, + "\uf0b5": 221, + "\uf0b6": 222, + "\uf0bb": 223, + "\uf8ef": 224, + "\uff05": 225, + "\uff0e": 226 + }, + "vocab_tag": { + "": 0, + "B-": 1, + "B-": 2, + "B-": 3, + "B-": 4, + "B-": 5, + "B-": 6, + "I-": 7, + "I-": 8, + "I-": 9, + "I-": 10, + "I-": 11, + "I-": 12, + "O": 13 + }, + "vocab_case": [ + "", + "numeric", + "allLower", + "allUpper", + "initialUpper", + "other", + "mainly_numeric", + "contains_digit" + ], + "max_char_length": 30, + "feature_preprocessor": { + "features_vocabulary_size": 12, + "features_indices": [ + 9, + 10, + 11, + 12, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "features_map_to_index": { + "9": { + "ALLCAPS": 1, + "INITCAP": 2, + "NOCAPS": 3 + }, + "10": { + "ALLDIGIT": 13, + "ALLDIGITS": 14, + "CONTAINDIGIT": 15, + "NODIGIT": 16 + }, + "11": { + "0": 25, + "1": 26 + }, + "12": { + "COMMA": 37, + "DOT": 38, + "ENDBRACKET": 39, + "HYPHEN": 40, + "NOPUNCT": 41, + "OPENBRACKET": 42, + "PUNCT": 43, + "QUOTE": 44 + }, + "16": { + "DIFFERENTFONT": 49, + "SAMEFONT": 50, + "SAxMEFONT": 51 + }, + "17": { + "HIGHERFONT": 61, + "LOWERFONT": 62, + "SAMEFONTSIZE": 63 + }, + "18": { + "false": 73, + "true": 74 + }, + "19": { + "false": 85, + "true": 86 + }, + "20": { + "BASELINE": 97, + "SUBSCRIPT": 98, + "SUPERSCRIPT": 99 + }, + "21": { + "false": 109, + "true": 110 + } + } + }, + "indice_tag": { + "0": "", + "1": "B-", + "2": "B-", + "3": "B-", + "4": "B-", + "5": "B-", + "6": "B-", + "7": "I-", + "8": "I-", + "9": "I-", + "10": "I-", + "11": "I-", + "12": "I-", + "13": "O" + } +} \ No newline at end of file diff --git a/resources/models/material-BERT_CRF/preprocessor.json b/resources/models/material-BERT_CRF/preprocessor.json new file mode 100644 index 00000000..261072bc --- /dev/null +++ b/resources/models/material-BERT_CRF/preprocessor.json @@ -0,0 +1,186 @@ +{ + "padding": true, + "return_lengths": false, + "return_word_embeddings": false, + "return_casing": false, + "return_features": false, + "return_chars": false, + "return_bert_embeddings": true, + "vocab_char": { + "": 0, + "": 1, + "\"": 2, + "$": 3, + "%": 4, + "'": 5, + "(": 6, + ")": 7, + "*": 8, + "+": 9, + ",": 10, + "-": 11, + ".": 12, + "/": 13, + "0": 14, + "1": 15, + "2": 16, + "3": 17, + "4": 18, + "5": 19, + "6": 20, + "7": 21, + "8": 22, + "9": 23, + ":": 24, + ";": 25, + "<": 26, + "=": 27, + ">": 28, + "A": 29, + "B": 30, + "C": 31, + "D": 32, + "E": 33, + "F": 34, + "G": 35, + "H": 36, + "I": 37, + "K": 38, + "L": 39, + "M": 40, + "N": 41, + "O": 42, + "P": 43, + "Q": 44, + "R": 45, + "S": 46, + "T": 47, + "U": 48, + "V": 49, + "W": 50, + "X": 51, + "Y": 52, + "Z": 53, + "[": 54, + "]": 55, + "_": 56, + "a": 57, + "b": 58, + "c": 59, + "d": 60, + "e": 61, + "f": 62, + "g": 63, + "h": 64, + "i": 65, + "j": 66, + "k": 67, + "l": 68, + "m": 69, + "n": 70, + "o": 71, + "p": 72, + "q": 73, + "r": 74, + "s": 75, + "t": 76, + "u": 77, + "v": 78, + "w": 79, + "x": 80, + "y": 81, + "z": 82, + "~": 83, + "\u00b1": 84, + "\u00bc": 85, + "\u00c0": 86, + "\u00c5": 87, + "\u00d7": 88, + "\u00e9": 89, + "\u0351": 90, + "\u0352": 91, + "\u03b1": 92, + "\u03b2": 93, + "\u03b3": 94, + "\u03b4": 95, + "\u03ba": 96, + "\u03ea": 97, + "\u03ed": 98, + "\u0440": 99, + "\u060a": 100, + "\u2022": 101, + "\u2032": 102, + "\u2190": 103, + "\u2192": 104, + "\u2194": 105, + "\u2206": 106, + "\u2212": 107, + "\u223c": 108, + "\u2243": 109, + "\u2248": 110, + "\u2264": 111, + "\u2265": 112, + "\u22c5": 113, + "\u2426": 114, + "\uf02d": 115, + "\uf03e": 116, + "\uf064": 117, + "\uf06c": 118, + "\uf0a0": 119, + "\uf0b3": 120, + "\uf0bb": 121, + "\uff0e": 122 + }, + "vocab_tag": { + "": 0, + "B-": 1, + "B-": 2, + "B-": 3, + "B-": 4, + "B-": 5, + "B-": 6, + "B-": 7, + "B-": 8, + "I-": 9, + "I-": 10, + "I-": 11, + "I-": 12, + "I-": 13, + "I-": 14, + "I-": 15, + "I-": 16, + "O": 17 + }, + "vocab_case": [ + "", + "numeric", + "allLower", + "allUpper", + "initialUpper", + "other", + "mainly_numeric", + "contains_digit" + ], + "max_char_length": 30, + "feature_preprocessor": null, + "indice_tag": { + "0": "", + "1": "B-", + "2": "B-", + "3": "B-", + "4": "B-", + "5": "B-", + "6": "B-", + "7": "B-", + "8": "B-", + "9": "I-", + "10": "I-", + "11": "I-", + "12": "I-", + "13": "I-", + "14": "I-", + "15": "I-", + "16": "I-", + "17": "O" + } +} \ No newline at end of file diff --git a/resources/web b/resources/web index 9ad40ec5..de30f6ca 160000 --- a/resources/web +++ b/resources/web @@ -1 +1 @@ -Subproject commit 9ad40ec5cc5373b1bb00d9156f66724c131627cb +Subproject commit de30f6cab109d12c3ae106b41a252b31ca939640 diff --git a/src/main/java/org/grobid/core/analyzers/DeepAnalyzer.java b/src/main/java/org/grobid/core/analyzers/DeepAnalyzer.java index 34d1df25..c5ad7293 100644 --- a/src/main/java/org/grobid/core/analyzers/DeepAnalyzer.java +++ b/src/main/java/org/grobid/core/analyzers/DeepAnalyzer.java @@ -1,5 +1,6 @@ package org.grobid.core.analyzers; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.grobid.core.lang.Language; import org.grobid.core.layout.LayoutToken; @@ -88,6 +89,21 @@ public List tokenizeWithLayoutToken(String text) { return result; } + @Override + public List retokenizeSubdigits(List list) { + throw new NotImplementedException("Not yet implemented"); + } + + @Override + public List retokenizeSubdigitsWithLayoutToken(List list) { + throw new NotImplementedException("Not yet implemented"); + } + + @Override + public List retokenizeSubdigitsFromLayoutToken(List list) { + throw new NotImplementedException("Not yet implemented"); + } + public List retokenize(List chunks) { List result = new ArrayList<>(); for (String chunk : chunks) { diff --git a/src/main/java/org/grobid/core/data/document/DocumentBlock.java b/src/main/java/org/grobid/core/data/document/DocumentBlock.java index 664c2229..8162c447 100644 --- a/src/main/java/org/grobid/core/data/document/DocumentBlock.java +++ b/src/main/java/org/grobid/core/data/document/DocumentBlock.java @@ -54,6 +54,11 @@ public DocumentBlock(List layoutTokens, String section, String subS this.markers = new ArrayList<>(markers); } + public DocumentBlock(List layoutTokens, String section, String subSection, String groupId, List spanList, List> markers) { + this(layoutTokens, section, subSection, spanList, markers); + this.groupId = groupId; + } + public String getSubSection() { return subSection; diff --git a/src/main/java/org/grobid/core/data/material/ChemicalComposition.java b/src/main/java/org/grobid/core/data/material/ChemicalComposition.java index f64e71b9..878bc190 100644 --- a/src/main/java/org/grobid/core/data/material/ChemicalComposition.java +++ b/src/main/java/org/grobid/core/data/material/ChemicalComposition.java @@ -12,6 +12,10 @@ public class ChemicalComposition { private String formula; private String name; + + private Integer code; + + private String message; public String getFormula() { return formula; @@ -38,7 +42,7 @@ public void setComposition(Map composition) { } public boolean isEmpty() { - return StringUtils.isBlank(formula) && StringUtils.isBlank(name) && composition.keySet().size() == 0; + return StringUtils.isBlank(formula) && StringUtils.isBlank(name) && composition.keySet().isEmpty(); } @Override @@ -62,4 +66,20 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(composition, formula, name); } + + public Integer getCode() { + return code; + } + + public void setCode(Integer code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } } diff --git a/src/main/java/org/grobid/core/data/material/Material.java b/src/main/java/org/grobid/core/data/material/Material.java index 425fffc9..866ed565 100644 --- a/src/main/java/org/grobid/core/data/material/Material.java +++ b/src/main/java/org/grobid/core/data/material/Material.java @@ -182,7 +182,7 @@ public static List resolveVariables(Material material) { // } try { - generatePermutations(mapOfContainedVariables, new ArrayList(containedVariables), outputFormulasString, Pair.of(0, 0), material.getFormula().getRawValue()); + generatePermutations(mapOfContainedVariables, new ArrayList<>(containedVariables), outputFormulasString, Pair.of(0, 0), material.getFormula().getRawValue()); } catch (NumberFormatException e) { Map> cleanedMapOfContainedVariables = new HashMap<>(); @@ -194,7 +194,7 @@ public static List resolveVariables(Material material) { }); try { - generatePermutations(cleanedMapOfContainedVariables, new ArrayList(containedVariables), outputFormulasString, Pair.of(0, 0), material.getFormula().getRawValue()); + generatePermutations(cleanedMapOfContainedVariables, new ArrayList<>(containedVariables), outputFormulasString, Pair.of(0, 0), material.getFormula().getRawValue()); } catch (NumberFormatException e2) { LOGGER.debug("Cannot replace variables " + Arrays.toString(variables.toArray())); } diff --git a/src/main/java/org/grobid/core/engines/GrobidPDFEngine.java b/src/main/java/org/grobid/core/engines/GrobidPDFEngine.java index 2a7c1f99..dfabb7f6 100644 --- a/src/main/java/org/grobid/core/engines/GrobidPDFEngine.java +++ b/src/main/java/org/grobid/core/engines/GrobidPDFEngine.java @@ -2,6 +2,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Sets; +import org.apache.commons.codec.digest.Md5Crypt; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; @@ -25,11 +26,13 @@ import org.grobid.core.tokenization.TaggingTokenCluster; import org.grobid.core.tokenization.TaggingTokenClusteror; import org.grobid.core.utilities.AdditionalLayoutTokensUtil; +import org.grobid.core.utilities.LayoutTokensUtil; import org.grobid.core.utilities.OffsetPosition; import org.grobid.core.utilities.SentenceUtilities; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -299,7 +302,10 @@ public static BiblioInfo processDocument(Document doc, GrobidAnalysisConfig conf List documentBlocksBySentences = new ArrayList<>(); documentBlocks.stream().forEach(documentBlock -> { - String paragraphId = RandomStringUtils.random(10, true, true); + +// String paragraphMd5 = Md5Crypt.md5Crypt(LayoutTokensUtil.toText(documentBlock.getLayoutTokens()).getBytes(StandardCharsets.UTF_8)).substring(0, 10); + + String paragraphId = RandomStringUtils.random(10, true, true); List> markersExtremitiesAsIndex = new ArrayList<>(); List markersPositionsAsOffsetsInText = new ArrayList<>(); @@ -314,7 +320,7 @@ public static BiblioInfo processDocument(Document doc, GrobidAnalysisConfig conf .stream() .sorted(Comparator.comparingInt(AdditionalLayoutTokensUtil::getLayoutTokenListStartOffset)) .collect(Collectors.toList()); - + if (!sortedMarkersListByStartOffsets.equals(documentBlock.getMarkers())) { documentBlock.setMarkers(sortedMarkersListByStartOffsets); } diff --git a/src/main/java/org/grobid/core/engines/MaterialClassResolver.java b/src/main/java/org/grobid/core/engines/MaterialClassResolver.java index 3a68a660..6900aefd 100644 --- a/src/main/java/org/grobid/core/engines/MaterialClassResolver.java +++ b/src/main/java/org/grobid/core/engines/MaterialClassResolver.java @@ -1,5 +1,6 @@ package org.grobid.core.engines; +import com.google.inject.Inject; import com.google.inject.Singleton; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -9,7 +10,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; import java.util.Arrays; import java.util.Collections; import java.util.List; diff --git a/src/main/java/org/grobid/core/engines/MaterialParser.java b/src/main/java/org/grobid/core/engines/MaterialParser.java index 52ea4700..f50f96db 100644 --- a/src/main/java/org/grobid/core/engines/MaterialParser.java +++ b/src/main/java/org/grobid/core/engines/MaterialParser.java @@ -2,6 +2,7 @@ import com.google.common.collect.Iterables; +import com.google.inject.Inject; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.text.StringEscapeUtils; @@ -12,6 +13,7 @@ import org.grobid.core.data.material.Material; import org.grobid.core.engines.label.TaggingLabel; import org.grobid.core.exceptions.GrobidException; +import org.grobid.core.exceptions.GrobidExceptionStatus; import org.grobid.core.features.FeaturesVectorMaterial; import org.grobid.core.layout.BoundingBox; import org.grobid.core.layout.LayoutToken; @@ -26,12 +28,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import java.util.*; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.apache.commons.collections4.CollectionUtils.isEmpty; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; @@ -87,6 +87,84 @@ public List process(String text) { return process(SuperconductorsParser.textToLayoutTokens(text)); } + public List> processParallel(List texts) { + + List emptyIndices = IntStream.range(0, texts.size()) + .filter(i -> StringUtils.isBlank(texts.get(i))) + .boxed() + .toList(); + + Set emptyIndicesSet = new HashSet<>(emptyIndices); + List textsCopy = new ArrayList<>(); + + for (int is = 0; is < texts.size(); is++) { + if (!emptyIndices.contains(is)) { + textsCopy.add(texts.get(is)); + } + } + + List> asLayoutTokens = textsCopy.stream() + .map(SuperconductorsParser::textToLayoutTokens) + .collect(Collectors.toList()); + + List> processed = processParallelLT(asLayoutTokens); + + emptyIndices.forEach(i -> processed.add(i, new ArrayList<>())); + + List> output = new ArrayList<>(); + for (int is = 0; is < texts.size(); is++) { + if (!emptyIndices.contains(is)) { + output.add(processed.get(is)); + } else { + output.add(new ArrayList<>()); + } + } + + return output; + } + + public List> processParallelLT(List> layoutTokensBatch) { + + List> entities = new ArrayList<>(); + + //Normalisation + List> normalisedTokens = layoutTokensBatch.stream() + .map(SuperconductorsParser::normalizeAndRetokenizeLayoutTokens) + .toList(); + + try { + List tokensWithFeatures = normalisedTokens.stream().map(nt -> addFeatures(nt) + "\n").toList(); + + String labellingResult = null; + try { + labellingResult = label(tokensWithFeatures); + } catch (Exception e) { + throw new GrobidException("CRF labeling for superconductors parsing failed.", e); + } + + List resultingBlocks = Arrays.asList(labellingResult.split("\n\n")); + List> localEntities = extractParallelResults(normalisedTokens, resultingBlocks); + + entities.addAll(localEntities); + } catch (Exception e) { + throw new GrobidException("An exception occurred while running Grobid.", e); + } + + return entities; + } + + public List> extractParallelResults(List> tokens, List results) { + List> spans = new ArrayList<>(); + if (tokens.size() != results.size()) { + throw new GrobidException("One of the text provided is invalid or empty and cannot be tagged. Please provide a clean input.", GrobidExceptionStatus.BAD_INPUT_DATA); + } + for (int i = 0; i < tokens.size(); i++) { + spans.add(extractResults(tokens.get(i), results.get(i))); + } + + return spans; + } + public List process(List tokens) { diff --git a/src/main/java/org/grobid/core/engines/ModuleEngine.java b/src/main/java/org/grobid/core/engines/ModuleEngine.java index 298cda17..e98bdfda 100644 --- a/src/main/java/org/grobid/core/engines/ModuleEngine.java +++ b/src/main/java/org/grobid/core/engines/ModuleEngine.java @@ -1,6 +1,8 @@ package org.grobid.core.engines; import com.google.common.collect.Iterables; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -22,8 +24,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.io.File; import java.io.InputStream; import java.util.*; diff --git a/src/main/java/org/grobid/core/engines/RuleBasedLinker.java b/src/main/java/org/grobid/core/engines/RuleBasedLinker.java index 9cd74695..93326e67 100644 --- a/src/main/java/org/grobid/core/engines/RuleBasedLinker.java +++ b/src/main/java/org/grobid/core/engines/RuleBasedLinker.java @@ -1,5 +1,6 @@ package org.grobid.core.engines; +import com.google.inject.Inject; import com.google.inject.Singleton; import org.grobid.core.data.document.Link; import org.grobid.core.data.document.Span; @@ -9,7 +10,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/src/main/java/org/grobid/core/engines/SuperconductorsParser.java b/src/main/java/org/grobid/core/engines/SuperconductorsParser.java index 2930d09e..c2c32cca 100644 --- a/src/main/java/org/grobid/core/engines/SuperconductorsParser.java +++ b/src/main/java/org/grobid/core/engines/SuperconductorsParser.java @@ -1,6 +1,8 @@ package org.grobid.core.engines; import com.google.common.collect.Iterables; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -27,8 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.util.*; import java.util.stream.Collectors; diff --git a/src/main/java/org/grobid/core/engines/linking/CRFBasedLinker.java b/src/main/java/org/grobid/core/engines/linking/CRFBasedLinker.java index 7b98e683..1d10ba3e 100644 --- a/src/main/java/org/grobid/core/engines/linking/CRFBasedLinker.java +++ b/src/main/java/org/grobid/core/engines/linking/CRFBasedLinker.java @@ -1,6 +1,8 @@ package org.grobid.core.engines.linking; import com.google.common.collect.Iterables; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -23,8 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.util.*; import java.util.stream.Collectors; diff --git a/src/main/java/org/grobid/core/engines/linking/EntityLinker_MaterialTcValue.java b/src/main/java/org/grobid/core/engines/linking/EntityLinker_MaterialTcValue.java index ac19625a..138016a3 100644 --- a/src/main/java/org/grobid/core/engines/linking/EntityLinker_MaterialTcValue.java +++ b/src/main/java/org/grobid/core/engines/linking/EntityLinker_MaterialTcValue.java @@ -1,5 +1,7 @@ package org.grobid.core.engines.linking; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.grobid.core.GrobidModel; import org.grobid.core.data.document.Span; import org.grobid.core.engines.AbstractParser; @@ -10,8 +12,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/src/main/java/org/grobid/core/engines/linking/EntityLinker_TcValueMe_Method.java b/src/main/java/org/grobid/core/engines/linking/EntityLinker_TcValueMe_Method.java index f4758162..316ac2b1 100644 --- a/src/main/java/org/grobid/core/engines/linking/EntityLinker_TcValueMe_Method.java +++ b/src/main/java/org/grobid/core/engines/linking/EntityLinker_TcValueMe_Method.java @@ -1,5 +1,7 @@ package org.grobid.core.engines.linking; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.grobid.core.GrobidModel; import org.grobid.core.data.document.Span; import org.grobid.core.engines.AbstractParser; @@ -10,8 +12,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/src/main/java/org/grobid/core/engines/linking/EntityLinker_TcValuePressure.java b/src/main/java/org/grobid/core/engines/linking/EntityLinker_TcValuePressure.java index 1398df07..92779b1e 100644 --- a/src/main/java/org/grobid/core/engines/linking/EntityLinker_TcValuePressure.java +++ b/src/main/java/org/grobid/core/engines/linking/EntityLinker_TcValuePressure.java @@ -1,5 +1,7 @@ package org.grobid.core.engines.linking; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.grobid.core.GrobidModel; import org.grobid.core.data.document.Span; import org.grobid.core.engines.AbstractParser; @@ -10,8 +12,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/src/main/java/org/grobid/core/engines/training/SuperconductorsParserTrainingData.java b/src/main/java/org/grobid/core/engines/training/SuperconductorsParserTrainingData.java index 34f3831f..994be7d2 100644 --- a/src/main/java/org/grobid/core/engines/training/SuperconductorsParserTrainingData.java +++ b/src/main/java/org/grobid/core/engines/training/SuperconductorsParserTrainingData.java @@ -166,6 +166,7 @@ private void createTrainingPDF(File file, String outputDirectory, TrainingOutput DocumentBlock newDocumentBlock = new DocumentBlock(documentBlock); newDocumentBlock.setLayoutTokens(normalisedLayoutTokens); newDocumentBlock.setSpans(sortedEntities); + newDocumentBlock.setGroupId(documentBlock.getGroupId()); labeledTextList.add(newDocumentBlock); diff --git a/src/main/java/org/grobid/core/engines/training/SuperconductorsTrainingXMLFormatter.java b/src/main/java/org/grobid/core/engines/training/SuperconductorsTrainingXMLFormatter.java index 6d06987d..4b113755 100644 --- a/src/main/java/org/grobid/core/engines/training/SuperconductorsTrainingXMLFormatter.java +++ b/src/main/java/org/grobid/core/engines/training/SuperconductorsTrainingXMLFormatter.java @@ -35,20 +35,31 @@ public String format(List documentBlocks, int id) { Element body = teiElement("body"); +// Map> byParagraphs = documentBlocks.stream() +// .collect(Collectors.groupingBy(DocumentBlock::getParagraphId)); + + String previousParagraphId = null; + String previousSection = "NO_SECTION"; + Element previousParent = null; + Element parent = null; for (DocumentBlock block : documentBlocks) { + String paragraphId = block.getGroupId(); if (block.getSection().equals(DocumentBlock.SECTION_BODY)) { if (block.getSubSection().equals(DocumentBlock.SUB_SECTION_FIGURE)) { - body.appendChild(trainingExtraction(block.getSpans(), - block.getLayoutTokens(), "ab", Pair.of("type", "figureCaption"))); + parent = getParentElement(body, previousParagraphId, paragraphId, previousParent, "ab", Pair.of("type", "figureCaption")); + parent.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens(), "s")); } else if (block.getSubSection().equals(DocumentBlock.SUB_SECTION_TABLE)) { - body.appendChild(trainingExtraction(block.getSpans(), - block.getLayoutTokens(), "ab", Pair.of("type", "tableCaption"))); + parent = getParentElement(body, previousParagraphId, paragraphId, previousParent, "ab", Pair.of("type", "tableCaption")); + parent.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens(), "s")); } else if (block.getSubSection().equals(DocumentBlock.SUB_SECTION_PARAGRAPH)) { - body.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens())); + parent = getParentElement(body, previousParagraphId, paragraphId, previousParent, "p", null); + parent.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens(), "s")); } else if (block.getSubSection().equals(DocumentBlock.SUB_SECTION_TITLE_SECTION)) { - body.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens(), "head")); + parent = getParentElement(body, previousParagraphId, paragraphId, previousParent, "head", null); + parent.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens(), "s")); } else { - body.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens())); + parent = getParentElement(body, previousParagraphId, paragraphId, previousParent, "p", null); + parent.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens(), "s")); } } else if (block.getSection().equals(DocumentBlock.SECTION_HEADER)) { if (block.getSubSection().equals(DocumentBlock.SUB_SECTION_TITLE)) { @@ -65,25 +76,33 @@ public String format(List documentBlocks, int id) { throw new RuntimeException("new keywords, but no space for them... "); } } else if (block.getSubSection().equals(DocumentBlock.SUB_SECTION_ABSTRACT)) { - Element abstractElement = SuperconductorsTeiUtils.getElement(profileDesc, "abstract"); if (abstractElement == null) { abstractElement = teiElement("abstract"); profileDesc.appendChild(abstractElement); } - abstractElement.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens())); + parent = getParentElement(abstractElement, previousParagraphId, paragraphId, previousParent, "p", null); + parent.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens(), "s")); } else { throw new RuntimeException("The section or subsection have the wrong name. " + "This will cause loss of data in the output generated files. Section name: " + block.getSection() + ", " + block.getSubSection()); } } else if (block.getSection().equals(DocumentBlock.SECTION_ANNEX)) { - body.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens())); + if (!StringUtils.equals(paragraphId, previousParagraphId)) { + parent = teiElement("p"); + body.appendChild(parent); + } else { + parent = previousParent; + } + parent.appendChild(trainingExtraction(block.getSpans(), block.getLayoutTokens())); } else { throw new RuntimeException("The section or subsection have the wrong name. " + "This will cause loss of data in the output generated files. Section name: " + block.getSection() + ", " + block.getSubSection()); } + previousParent = parent; + previousParagraphId = paragraphId; } textNode.appendChild(body); @@ -91,6 +110,26 @@ public String format(List documentBlocks, int id) { return XmlBuilderUtils.toXml(outputDocumentRoot); } + /** + * Create the parent element or, if under certain conditions, recycle the previous one. + */ + protected Element getParentElement(Element body, String previousParagraphId, String paragraphId, Element previousParent, String parentTagName, Pair attributes) { + Element parent = null; + + if (previousParent == null || !StringUtils.equals(paragraphId, previousParagraphId)) { + parent = teiElement(parentTagName); + if (attributes != null) { + parent.addAttribute(new Attribute(attributes.getLeft(), attributes.getRight())); + } + body.appendChild(parent); + } else { + parent = previousParent; + } + + + return parent; + } + protected Element trainingExtraction(List spanList, List tokens) { return trainingExtraction(spanList, tokens, "p"); } @@ -125,7 +164,7 @@ protected Element trainingExtraction(List spanList, List toke p.appendChild(entityElement); // We stop the process if something doesn't match - int accumulatedOffset = startPosition + length(contentBefore) + length(name); + int accumulatedOffset = startPosition + length(contentBefore) + LayoutTokensUtil.toText(superconductor.getLayoutTokens()).stripTrailing().length(); if (end != accumulatedOffset) { throw new RuntimeException("Wrong synchronisation between entities and layout tokens. End entity offset: " + end + " different from the expected offset: " + accumulatedOffset); diff --git a/src/main/java/org/grobid/core/utilities/client/ChemDataExtractorClient.java b/src/main/java/org/grobid/core/utilities/client/ChemDataExtractorClient.java index 343aa4a4..76b9a0f5 100644 --- a/src/main/java/org/grobid/core/utilities/client/ChemDataExtractorClient.java +++ b/src/main/java/org/grobid/core/utilities/client/ChemDataExtractorClient.java @@ -6,6 +6,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -17,8 +19,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.io.IOException; import java.io.InputStream; import java.net.ConnectException; @@ -28,7 +28,7 @@ import java.util.List; import java.util.stream.Collectors; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; import static org.grobid.core.utilities.client.LinkingModuleClient.toJson_listOfString; @Singleton diff --git a/src/main/java/org/grobid/core/utilities/client/ChemicalMaterialParserClient.java b/src/main/java/org/grobid/core/utilities/client/ChemicalMaterialParserClient.java index 49e48857..cb711124 100644 --- a/src/main/java/org/grobid/core/utilities/client/ChemicalMaterialParserClient.java +++ b/src/main/java/org/grobid/core/utilities/client/ChemicalMaterialParserClient.java @@ -6,6 +6,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -18,8 +20,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -27,7 +27,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; @Singleton public class ChemicalMaterialParserClient { @@ -66,10 +66,16 @@ public ChemicalComposition convertNameToFormula(String name) { try (CloseableHttpResponse response = httpClient.execute(request)) { int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == HttpURLConnection.HTTP_OK) { - outputFormula = fromJsonToChemicalComposition(response.getEntity().getContent()); + if (statusCode != HttpURLConnection.HTTP_OK) { + LOGGER.debug("Not OK answer. Input: " + name + ". Status code: " + response.getStatusLine().getStatusCode()); } else { - LOGGER.debug("Not OK answer. Input: " + name + ", status code: " + statusCode); + outputFormula = fromJsonToChemicalComposition(response.getEntity().getContent()); + if (outputFormula != null && outputFormula.getCode() != HttpURLConnection.HTTP_OK) { + LOGGER.debug("Not OK answer. Input: " + name + ". " + + "Status code: " + outputFormula.getCode() + + "Message: " + outputFormula.getMessage()); + outputFormula = new ChemicalComposition(); + } } } @@ -101,6 +107,11 @@ public ChemicalComposition convertFormulaToComposition(String formula) { LOGGER.debug("Not OK answer. Input: " + formula + ". Status code: " + response.getStatusLine().getStatusCode()); } else { outputComposition = fromJsonToChemicalComposition(response.getEntity().getContent()); + if (outputComposition != null && outputComposition.getCode() != HttpURLConnection.HTTP_OK) { + LOGGER.debug("Not OK answer. Input: " + formula + ". Status code: " + outputComposition.getCode() + + "Message: " + outputComposition.getMessage()); + outputComposition = new ChemicalComposition(); + } } } diff --git a/src/main/java/org/grobid/core/utilities/client/ChemspotClient.java b/src/main/java/org/grobid/core/utilities/client/ChemspotClient.java index 0a375d3e..58599865 100644 --- a/src/main/java/org/grobid/core/utilities/client/ChemspotClient.java +++ b/src/main/java/org/grobid/core/utilities/client/ChemspotClient.java @@ -6,6 +6,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -17,8 +19,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -26,7 +26,8 @@ import java.util.ArrayList; import java.util.List; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + @Singleton public class ChemspotClient { diff --git a/src/main/java/org/grobid/core/utilities/client/ClassResolverModuleClient.java b/src/main/java/org/grobid/core/utilities/client/ClassResolverModuleClient.java index ef58bff6..0fcc12df 100644 --- a/src/main/java/org/grobid/core/utilities/client/ClassResolverModuleClient.java +++ b/src/main/java/org/grobid/core/utilities/client/ClassResolverModuleClient.java @@ -6,6 +6,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -17,8 +19,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.List; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; @Singleton public class ClassResolverModuleClient { diff --git a/src/main/java/org/grobid/core/utilities/client/GrobidQuantitiesClient.java b/src/main/java/org/grobid/core/utilities/client/GrobidQuantitiesClient.java index d1ca01a4..6f1e1b85 100644 --- a/src/main/java/org/grobid/core/utilities/client/GrobidQuantitiesClient.java +++ b/src/main/java/org/grobid/core/utilities/client/GrobidQuantitiesClient.java @@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.http.Consts; import org.apache.http.NameValuePair; import org.apache.http.client.entity.UrlEncodedFormEntity; @@ -17,8 +19,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -27,8 +27,9 @@ import java.util.LinkedHashMap; import java.util.List; -import static javax.ws.rs.core.MediaType.APPLICATION_FORM_URLENCODED; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.APPLICATION_FORM_URLENCODED; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + @Singleton public class GrobidQuantitiesClient { diff --git a/src/main/java/org/grobid/core/utilities/client/LinkingModuleClient.java b/src/main/java/org/grobid/core/utilities/client/LinkingModuleClient.java index 2dd1ee39..b69d6b39 100644 --- a/src/main/java/org/grobid/core/utilities/client/LinkingModuleClient.java +++ b/src/main/java/org/grobid/core/utilities/client/LinkingModuleClient.java @@ -6,6 +6,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -18,8 +20,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -27,7 +27,8 @@ import java.nio.charset.StandardCharsets; import java.util.List; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + @Singleton public class LinkingModuleClient { diff --git a/src/main/java/org/grobid/core/utilities/client/StructureIdentificationModuleClient.java b/src/main/java/org/grobid/core/utilities/client/StructureIdentificationModuleClient.java index 3a6a62ce..3b4ca914 100644 --- a/src/main/java/org/grobid/core/utilities/client/StructureIdentificationModuleClient.java +++ b/src/main/java/org/grobid/core/utilities/client/StructureIdentificationModuleClient.java @@ -6,6 +6,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -18,8 +20,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -28,7 +28,7 @@ import java.util.ArrayList; import java.util.List; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; @Singleton public class StructureIdentificationModuleClient { diff --git a/src/main/java/org/grobid/service/EmptyOptionalNoContentExceptionMapper.java b/src/main/java/org/grobid/service/EmptyOptionalNoContentExceptionMapper.java index d81ef81a..7fafdfce 100644 --- a/src/main/java/org/grobid/service/EmptyOptionalNoContentExceptionMapper.java +++ b/src/main/java/org/grobid/service/EmptyOptionalNoContentExceptionMapper.java @@ -2,9 +2,9 @@ import io.dropwizard.jersey.optional.EmptyOptionalException; import io.dropwizard.jersey.optional.EmptyOptionalExceptionMapper; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.ExceptionMapper; /** * Returns a 204 for Optional.empty() diff --git a/src/main/java/org/grobid/service/GrobidEngineInitialiser.java b/src/main/java/org/grobid/service/GrobidEngineInitialiser.java index db08eb96..f7c4018d 100644 --- a/src/main/java/org/grobid/service/GrobidEngineInitialiser.java +++ b/src/main/java/org/grobid/service/GrobidEngineInitialiser.java @@ -1,6 +1,8 @@ package org.grobid.service; import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.commons.lang3.StringUtils; import org.grobid.core.main.GrobidHomeFinder; import org.grobid.core.main.LibraryLoader; @@ -10,8 +12,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; import java.lang.reflect.Field; @Singleton diff --git a/src/main/java/org/grobid/service/GrobidSuperconductorsApplication.java b/src/main/java/org/grobid/service/GrobidSuperconductorsApplication.java index 7c9c6ac3..9e3528d1 100755 --- a/src/main/java/org/grobid/service/GrobidSuperconductorsApplication.java +++ b/src/main/java/org/grobid/service/GrobidSuperconductorsApplication.java @@ -1,25 +1,22 @@ package org.grobid.service; -import com.google.inject.Module; -import com.hubspot.dropwizard.guicier.GuiceBundle; -import io.dropwizard.Application; +import com.google.inject.AbstractModule; import io.dropwizard.assets.AssetsBundle; import io.dropwizard.configuration.EnvironmentVariableSubstitutor; import io.dropwizard.configuration.SubstitutingSourceProvider; +import io.dropwizard.core.Application; +import io.dropwizard.core.setup.Bootstrap; +import io.dropwizard.core.setup.Environment; import io.dropwizard.forms.MultiPartBundle; -import io.dropwizard.setup.Bootstrap; -import io.dropwizard.setup.Environment; +import jakarta.servlet.DispatcherType; +import jakarta.servlet.FilterRegistration; import org.eclipse.jetty.servlets.CrossOriginFilter; -import org.eclipse.jetty.servlets.DoSFilter; import org.eclipse.jetty.servlets.QoSFilter; import org.grobid.service.command.*; import org.grobid.service.configuration.GrobidSuperconductorsConfiguration; +import ru.vyarus.dropwizard.guice.GuiceBundle; -import javax.servlet.DispatcherType; -import javax.servlet.FilterRegistration; -import java.util.Arrays; import java.util.EnumSet; -import java.util.List; public class GrobidSuperconductorsApplication extends Application { private static final String RESOURCES = "/service"; @@ -33,8 +30,8 @@ public String getName() { return "grobid-superconductors"; } - private List getGuiceModules() { - return Arrays.asList(new SuperconductorsServiceModule()); + private AbstractModule getGuiceModules() { + return new SuperconductorsServiceModule(); } @Override @@ -42,9 +39,10 @@ public void initialize(Bootstrap bootstrap) bootstrap.setConfigurationSourceProvider(new SubstitutingSourceProvider( bootstrap.getConfigurationSourceProvider(), new EnvironmentVariableSubstitutor(false))); - GuiceBundle guiceBundle = GuiceBundle.defaultBuilder(GrobidSuperconductorsConfiguration.class) + GuiceBundle guiceBundle = GuiceBundle.builder() .modules(getGuiceModules()) .build(); + bootstrap.addBundle(guiceBundle); bootstrap.addBundle(new MultiPartBundle()); bootstrap.addBundle(new AssetsBundle("/web", "/", "index.html", "assets")); diff --git a/src/main/java/org/grobid/service/SuperconductorsServiceModule.java b/src/main/java/org/grobid/service/SuperconductorsServiceModule.java index b9bdc522..b16280c0 100644 --- a/src/main/java/org/grobid/service/SuperconductorsServiceModule.java +++ b/src/main/java/org/grobid/service/SuperconductorsServiceModule.java @@ -1,10 +1,8 @@ package org.grobid.service; -import com.codahale.metrics.MetricRegistry; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.inject.Binder; import com.google.inject.Provides; -import com.hubspot.dropwizard.guicier.DropwizardAwareModule; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; import org.grobid.core.engines.*; import org.grobid.core.engines.linking.CRFBasedLinker; import org.grobid.core.utilities.client.*; @@ -14,59 +12,42 @@ import org.grobid.service.exceptions.mapper.GrobidExceptionsTranslationUtility; import org.grobid.service.exceptions.mapper.GrobidServiceExceptionMapper; import org.grobid.service.exceptions.mapper.WebApplicationExceptionMapper; - -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; +import ru.vyarus.dropwizard.guice.module.support.DropwizardAwareModule; public class SuperconductorsServiceModule extends DropwizardAwareModule { @Override - public void configure(Binder binder) { + public void configure() { // Generic modules - binder.bind(GrobidEngineInitialiser.class); - binder.bind(HealthCheck.class); + bind(GrobidEngineInitialiser.class); + bind(HealthCheck.class); //Core services - binder.bind(ChemspotClient.class); - binder.bind(ChemDataExtractorClient.class); - binder.bind(StructureIdentificationModuleClient.class); - binder.bind(ClassResolverModuleClient.class); - binder.bind(MaterialClassResolver.class); - binder.bind(ChemicalMaterialParserClient.class); - binder.bind(MaterialParser.class); - binder.bind(LinkingModuleClient.class); - binder.bind(RuleBasedLinker.class); - binder.bind(CRFBasedLinker.class); - binder.bind(SuperconductorsParser.class); - binder.bind(ModuleEngine.class); + bind(ChemspotClient.class); + bind(ChemDataExtractorClient.class); + bind(StructureIdentificationModuleClient.class); + bind(ClassResolverModuleClient.class); + bind(MaterialClassResolver.class); + bind(ChemicalMaterialParserClient.class); + bind(MaterialParser.class); + bind(LinkingModuleClient.class); + bind(RuleBasedLinker.class); + bind(CRFBasedLinker.class); + bind(SuperconductorsParser.class); + bind(ModuleEngine.class); //REST - binder.bind(AnnotationController.class); - binder.bind(MaterialController.class); - binder.bind(LinkerController.class); - binder.bind(ServiceController.class); + bind(AnnotationController.class); + bind(MaterialController.class); + bind(LinkerController.class); + bind(ServiceController.class); //Exception Mappers - binder.bind(GrobidServiceExceptionMapper.class); - binder.bind(GrobidExceptionsTranslationUtility.class); - binder.bind(GrobidExceptionMapper.class); - binder.bind(WebApplicationExceptionMapper.class); - } - - @Provides - protected ObjectMapper getObjectMapper() { - return getEnvironment().getObjectMapper(); - } - - @Provides - protected MetricRegistry provideMetricRegistry() { - return getMetricRegistry(); - } - - //for unit tests - protected MetricRegistry getMetricRegistry() { - return getEnvironment().metrics(); + bind(GrobidServiceExceptionMapper.class); + bind(GrobidExceptionsTranslationUtility.class); + bind(GrobidExceptionMapper.class); + bind(WebApplicationExceptionMapper.class); } @Provides diff --git a/src/main/java/org/grobid/service/command/InterAnnotationAgreementCommand.java b/src/main/java/org/grobid/service/command/InterAnnotationAgreementCommand.java index 2351e268..19db78d8 100644 --- a/src/main/java/org/grobid/service/command/InterAnnotationAgreementCommand.java +++ b/src/main/java/org/grobid/service/command/InterAnnotationAgreementCommand.java @@ -1,7 +1,7 @@ package org.grobid.service.command; -import io.dropwizard.cli.ConfiguredCommand; -import io.dropwizard.setup.Bootstrap; +import io.dropwizard.core.cli.ConfiguredCommand; +import io.dropwizard.core.setup.Bootstrap; import net.sourceforge.argparse4j.impl.Arguments; import net.sourceforge.argparse4j.inf.Namespace; import net.sourceforge.argparse4j.inf.Subparser; diff --git a/src/main/java/org/grobid/service/command/PrepareDelftTrainingCommand.java b/src/main/java/org/grobid/service/command/PrepareDelftTrainingCommand.java index 3af119ea..df0828a1 100644 --- a/src/main/java/org/grobid/service/command/PrepareDelftTrainingCommand.java +++ b/src/main/java/org/grobid/service/command/PrepareDelftTrainingCommand.java @@ -1,7 +1,7 @@ package org.grobid.service.command; -import io.dropwizard.cli.ConfiguredCommand; -import io.dropwizard.setup.Bootstrap; +import io.dropwizard.core.cli.ConfiguredCommand; +import io.dropwizard.core.setup.Bootstrap; import net.sourceforge.argparse4j.impl.Arguments; import net.sourceforge.argparse4j.inf.MutuallyExclusiveGroup; import net.sourceforge.argparse4j.inf.Namespace; diff --git a/src/main/java/org/grobid/service/command/PrepareMaterialParserTrainingCommand.java b/src/main/java/org/grobid/service/command/PrepareMaterialParserTrainingCommand.java index 1ab48bfa..b7944cfa 100644 --- a/src/main/java/org/grobid/service/command/PrepareMaterialParserTrainingCommand.java +++ b/src/main/java/org/grobid/service/command/PrepareMaterialParserTrainingCommand.java @@ -1,7 +1,7 @@ package org.grobid.service.command; -import io.dropwizard.cli.ConfiguredCommand; -import io.dropwizard.setup.Bootstrap; +import io.dropwizard.core.cli.ConfiguredCommand; +import io.dropwizard.core.setup.Bootstrap; import net.sourceforge.argparse4j.impl.Arguments; import net.sourceforge.argparse4j.inf.Namespace; import net.sourceforge.argparse4j.inf.Subparser; diff --git a/src/main/java/org/grobid/service/command/RunTrainingCommand.java b/src/main/java/org/grobid/service/command/RunTrainingCommand.java index fedc4cdb..4998b75d 100644 --- a/src/main/java/org/grobid/service/command/RunTrainingCommand.java +++ b/src/main/java/org/grobid/service/command/RunTrainingCommand.java @@ -1,7 +1,7 @@ package org.grobid.service.command; -import io.dropwizard.cli.ConfiguredCommand; -import io.dropwizard.setup.Bootstrap; +import io.dropwizard.core.cli.ConfiguredCommand; +import io.dropwizard.core.setup.Bootstrap; import net.sourceforge.argparse4j.inf.Namespace; import net.sourceforge.argparse4j.inf.Subparser; import org.grobid.core.engines.SuperconductorsModels; diff --git a/src/main/java/org/grobid/service/command/TrainingGenerationCommand.java b/src/main/java/org/grobid/service/command/TrainingGenerationCommand.java index 67e681fd..359707d1 100644 --- a/src/main/java/org/grobid/service/command/TrainingGenerationCommand.java +++ b/src/main/java/org/grobid/service/command/TrainingGenerationCommand.java @@ -1,7 +1,7 @@ package org.grobid.service.command; -import io.dropwizard.cli.ConfiguredCommand; -import io.dropwizard.setup.Bootstrap; +import io.dropwizard.core.cli.ConfiguredCommand; +import io.dropwizard.core.setup.Bootstrap; import net.sourceforge.argparse4j.impl.Arguments; import net.sourceforge.argparse4j.inf.Namespace; import net.sourceforge.argparse4j.inf.Subparser; diff --git a/src/main/java/org/grobid/service/configuration/GrobidSuperconductorsConfiguration.java b/src/main/java/org/grobid/service/configuration/GrobidSuperconductorsConfiguration.java index 12c20dd0..4f5bf679 100755 --- a/src/main/java/org/grobid/service/configuration/GrobidSuperconductorsConfiguration.java +++ b/src/main/java/org/grobid/service/configuration/GrobidSuperconductorsConfiguration.java @@ -1,12 +1,12 @@ package org.grobid.service.configuration; import com.fasterxml.jackson.annotation.JsonProperty; -import io.dropwizard.Configuration; +import io.dropwizard.core.Configuration; import org.apache.commons.io.IOUtils; -import org.grobid.core.utilities.GrobidConfig; -import org.grobid.core.utilities.GrobidProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.grobid.core.utilities.GrobidConfig; +import org.grobid.core.utilities.GrobidProperties; import java.io.IOException; import java.io.InputStream; diff --git a/src/main/java/org/grobid/service/controller/AnnotationController.java b/src/main/java/org/grobid/service/controller/AnnotationController.java index 81966d42..750175b7 100644 --- a/src/main/java/org/grobid/service/controller/AnnotationController.java +++ b/src/main/java/org/grobid/service/controller/AnnotationController.java @@ -1,5 +1,7 @@ package org.grobid.service.controller; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -12,10 +14,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.MediaType; import java.io.InputStream; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/org/grobid/service/controller/HealthCheck.java b/src/main/java/org/grobid/service/controller/HealthCheck.java index 83851406..202b6612 100644 --- a/src/main/java/org/grobid/service/controller/HealthCheck.java +++ b/src/main/java/org/grobid/service/controller/HealthCheck.java @@ -1,15 +1,16 @@ package org.grobid.service.controller; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.grobid.service.configuration.GrobidSuperconductorsConfiguration; -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.Response; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.Response; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; @Path("health") @Singleton diff --git a/src/main/java/org/grobid/service/controller/LinkerController.java b/src/main/java/org/grobid/service/controller/LinkerController.java index 218ea02f..c982a1e2 100644 --- a/src/main/java/org/grobid/service/controller/LinkerController.java +++ b/src/main/java/org/grobid/service/controller/LinkerController.java @@ -1,6 +1,8 @@ package org.grobid.service.controller; import com.ctc.wstx.stax.WstxInputFactory; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.commons.lang3.tuple.Pair; import org.codehaus.stax2.XMLStreamReader2; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -14,14 +16,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import javax.xml.stream.XMLStreamException; import java.io.ByteArrayInputStream; import java.io.InputStream; diff --git a/src/main/java/org/grobid/service/controller/MaterialController.java b/src/main/java/org/grobid/service/controller/MaterialController.java index c9155478..dfbdf309 100644 --- a/src/main/java/org/grobid/service/controller/MaterialController.java +++ b/src/main/java/org/grobid/service/controller/MaterialController.java @@ -1,17 +1,18 @@ package org.grobid.service.controller; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; import org.glassfish.jersey.media.multipart.FormDataParam; import org.grobid.core.data.material.Material; import org.grobid.core.engines.MaterialParser; import org.grobid.service.configuration.GrobidSuperconductorsConfiguration; -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; +import java.util.Arrays; import java.util.List; @Singleton @@ -42,10 +43,32 @@ public List processTextSuperconductorsGet(@FormDataParam("text") Strin return parseMaterial(text); } + + @Path("multiparse") + @Produces(MediaType.APPLICATION_JSON) + @POST + public List> processTextSuperconductorsPost2(@FormDataParam("texts") String texts) { + return parseMaterials(texts); + } + + @Path("multiparse") + @Produces(MediaType.APPLICATION_JSON) + @GET + public List> processTextSuperconductorsGet2(@FormDataParam("texts") String texts) { + return parseMaterials(texts); + } + private List parseMaterial(@FormDataParam("text") String text) { String textPreprocessed = text.replace("\r\n", "\n"); return materialParser.process(textPreprocessed); } + private List> parseMaterials(@FormDataParam("text") String text) { + String textPreprocessed = text.replace("\r\n", "\n"); + + List list = Arrays.asList(textPreprocessed.split("\n")); + return materialParser.processParallel(list); + } + } diff --git a/src/main/java/org/grobid/service/controller/ServiceController.java b/src/main/java/org/grobid/service/controller/ServiceController.java index 17fbfee9..92a0584e 100644 --- a/src/main/java/org/grobid/service/controller/ServiceController.java +++ b/src/main/java/org/grobid/service/controller/ServiceController.java @@ -1,14 +1,14 @@ package org.grobid.service.controller; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.grobid.core.data.ServiceInfo; import org.grobid.service.configuration.GrobidSuperconductorsConfiguration; -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; @Singleton @Path("/") diff --git a/src/main/java/org/grobid/service/exceptions/GrobidServiceException.java b/src/main/java/org/grobid/service/exceptions/GrobidServiceException.java index 86594f0d..8ff8d501 100755 --- a/src/main/java/org/grobid/service/exceptions/GrobidServiceException.java +++ b/src/main/java/org/grobid/service/exceptions/GrobidServiceException.java @@ -2,7 +2,7 @@ import org.grobid.core.exceptions.GrobidException; -import javax.ws.rs.core.Response; +import jakarta.ws.rs.core.Response; public class GrobidServiceException extends GrobidException { diff --git a/src/main/java/org/grobid/service/exceptions/GrobidServicePropertyException.java b/src/main/java/org/grobid/service/exceptions/GrobidServicePropertyException.java index 67d20b96..d0b3965a 100755 --- a/src/main/java/org/grobid/service/exceptions/GrobidServicePropertyException.java +++ b/src/main/java/org/grobid/service/exceptions/GrobidServicePropertyException.java @@ -1,6 +1,6 @@ package org.grobid.service.exceptions; -import javax.ws.rs.core.Response; +import jakarta.ws.rs.core.Response; @Deprecated public class GrobidServicePropertyException extends GrobidServiceException { diff --git a/src/main/java/org/grobid/service/exceptions/mapper/GrobidExceptionMapper.java b/src/main/java/org/grobid/service/exceptions/mapper/GrobidExceptionMapper.java index 0a0eff9e..30820b2d 100644 --- a/src/main/java/org/grobid/service/exceptions/mapper/GrobidExceptionMapper.java +++ b/src/main/java/org/grobid/service/exceptions/mapper/GrobidExceptionMapper.java @@ -5,12 +5,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriInfo; -import javax.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.ext.Provider; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import jakarta.ws.rs.ext.ExceptionMapper; +import jakarta.ws.rs.ext.Provider; @Provider public class GrobidExceptionMapper implements ExceptionMapper { diff --git a/src/main/java/org/grobid/service/exceptions/mapper/GrobidExceptionsTranslationUtility.java b/src/main/java/org/grobid/service/exceptions/mapper/GrobidExceptionsTranslationUtility.java index 64f103e0..92929b38 100644 --- a/src/main/java/org/grobid/service/exceptions/mapper/GrobidExceptionsTranslationUtility.java +++ b/src/main/java/org/grobid/service/exceptions/mapper/GrobidExceptionsTranslationUtility.java @@ -4,13 +4,13 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Joiner; import com.google.common.base.Throwables; +import com.google.inject.Inject; import org.grobid.core.exceptions.GrobidExceptionStatus; import org.slf4j.MDC; -import javax.inject.Inject; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.Provider; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.ext.Provider; import java.io.IOException; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/org/grobid/service/exceptions/mapper/GrobidServiceExceptionMapper.java b/src/main/java/org/grobid/service/exceptions/mapper/GrobidServiceExceptionMapper.java index ebe26ffb..7e92c1fe 100644 --- a/src/main/java/org/grobid/service/exceptions/mapper/GrobidServiceExceptionMapper.java +++ b/src/main/java/org/grobid/service/exceptions/mapper/GrobidServiceExceptionMapper.java @@ -5,12 +5,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriInfo; -import javax.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.ext.Provider; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import jakarta.ws.rs.ext.ExceptionMapper; +import jakarta.ws.rs.ext.Provider; @Provider public class GrobidServiceExceptionMapper implements ExceptionMapper { diff --git a/src/main/java/org/grobid/service/exceptions/mapper/GrobidStatusToHttpStatusMapper.java b/src/main/java/org/grobid/service/exceptions/mapper/GrobidStatusToHttpStatusMapper.java index 9642d13a..54a0a869 100644 --- a/src/main/java/org/grobid/service/exceptions/mapper/GrobidStatusToHttpStatusMapper.java +++ b/src/main/java/org/grobid/service/exceptions/mapper/GrobidStatusToHttpStatusMapper.java @@ -2,7 +2,7 @@ import org.grobid.core.exceptions.GrobidExceptionStatus; -import javax.ws.rs.core.Response; +import jakarta.ws.rs.core.Response; public class GrobidStatusToHttpStatusMapper { public static Response.Status getStatusCode(GrobidExceptionStatus status) { diff --git a/src/main/java/org/grobid/service/exceptions/mapper/WebApplicationExceptionMapper.java b/src/main/java/org/grobid/service/exceptions/mapper/WebApplicationExceptionMapper.java index 7d72e026..7489b427 100644 --- a/src/main/java/org/grobid/service/exceptions/mapper/WebApplicationExceptionMapper.java +++ b/src/main/java/org/grobid/service/exceptions/mapper/WebApplicationExceptionMapper.java @@ -2,10 +2,10 @@ import com.google.inject.Inject; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.ext.Provider; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.ext.ExceptionMapper; +import jakarta.ws.rs.ext.Provider; @Provider public class WebApplicationExceptionMapper implements ExceptionMapper { diff --git a/src/main/resources/web/index.html b/src/main/resources/web/index.html index a57532c0..535bd76e 100644 --- a/src/main/resources/web/index.html +++ b/src/main/resources/web/index.html @@ -7,7 +7,7 @@ - + @@ -445,7 +445,7 @@

",pt("type|href|height|width",dt,"#"===e.firstChild.getAttribute("href")),pt(B,ft,null==e.getAttribute("disabled")),e.className="i",!e.getAttribute("className")}),r.input=ct(function(e){return e.innerHTML="",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")}),pt("value",ht,r.attributes&&r.input),r.getElementsByTagName=ct(function(e){return e.appendChild(n.createComment("")),!e.getElementsByTagName("*").length}),r.getElementsByClassName=ct(function(e){return e.innerHTML="
",e.firstChild.className="i",2===e.getElementsByClassName("i").length}),r.getById=ct(function(e){return d.appendChild(e).id=b,!n.getElementsByName||!n.getElementsByName(b).length}),r.getById?(o.find.ID=function(e,t){if(typeof t.getElementById!==j&&h){var n=t.getElementById(e);return n&&n.parentNode?[n]:[]}},o.filter.ID=function(e){var t=e.replace(rt,it);return function(e){return e.getAttribute("id")===t}}):(delete o.find.ID,o.filter.ID=function(e){var t=e.replace(rt,it);return function(e){var n=typeof e.getAttributeNode!==j&&e.getAttributeNode("id");return n&&n.value===t}}),o.find.TAG=r.getElementsByTagName?function(e,n){return typeof n.getElementsByTagName!==j?n.getElementsByTagName(e):t}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},o.find.CLASS=r.getElementsByClassName&&function(e,n){return typeof n.getElementsByClassName!==j&&h?n.getElementsByClassName(e):t},m=[],g=[],(r.qsa=st(n.querySelectorAll))&&(ct(function(e){e.innerHTML="",e.querySelectorAll("[selected]").length||g.push("\\["+P+"*(?:value|"+B+")"),e.querySelectorAll(":checked").length||g.push(":checked")}),ct(function(e){var t=n.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("t",""),e.querySelectorAll("[t^='']").length&&g.push("[*^$]="+P+"*(?:''|\"\")"),e.querySelectorAll(":enabled").length||g.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),g.push(",.*:")})),(r.matchesSelector=st(y=d.webkitMatchesSelector||d.mozMatchesSelector||d.oMatchesSelector||d.msMatchesSelector))&&ct(function(e){r.disconnectedMatch=y.call(e,"div"),y.call(e,"[s!='']:x"),m.push("!=",I)}),g=g.length&&RegExp(g.join("|")),m=m.length&&RegExp(m.join("|")),v=st(d.contains)||d.compareDocumentPosition?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},r.sortDetached=ct(function(e){return 1&e.compareDocumentPosition(n.createElement("div"))}),A=d.compareDocumentPosition?function(e,t){if(e===t)return S=!0,0;var i=t.compareDocumentPosition&&e.compareDocumentPosition&&e.compareDocumentPosition(t);return i?1&i||!r.sortDetached&&t.compareDocumentPosition(e)===i?e===n||v(w,e)?-1:t===n||v(w,t)?1:c?F.call(c,e)-F.call(c,t):0:4&i?-1:1:e.compareDocumentPosition?-1:1}:function(e,t){var r,i=0,o=e.parentNode,a=t.parentNode,s=[e],l=[t];if(e===t)return S=!0,0;if(!o||!a)return e===n?-1:t===n?1:o?-1:a?1:c?F.call(c,e)-F.call(c,t):0;if(o===a)return gt(e,t);r=e;while(r=r.parentNode)s.unshift(r);r=t;while(r=r.parentNode)l.unshift(r);while(s[i]===l[i])i++;return i?gt(s[i],l[i]):s[i]===w?-1:l[i]===w?1:0},n):f},at.matches=function(e,t){return at(e,null,null,t)},at.matchesSelector=function(e,t){if((e.ownerDocument||e)!==f&&p(e),t=t.replace(Y,"='$1']"),!(!r.matchesSelector||!h||m&&m.test(t)||g&&g.test(t)))try{var n=y.call(e,t);if(n||r.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(i){}return at(t,f,null,[e]).length>0},at.contains=function(e,t){return(e.ownerDocument||e)!==f&&p(e),v(e,t)},at.attr=function(e,n){(e.ownerDocument||e)!==f&&p(e);var i=o.attrHandle[n.toLowerCase()],a=i&&L.call(o.attrHandle,n.toLowerCase())?i(e,n,!h):t;return a===t?r.attributes||!h?e.getAttribute(n):(a=e.getAttributeNode(n))&&a.specified?a.value:null:a},at.error=function(e){throw Error("Syntax error, unrecognized expression: "+e)},at.uniqueSort=function(e){var t,n=[],i=0,o=0;if(S=!r.detectDuplicates,c=!r.sortStable&&e.slice(0),e.sort(A),S){while(t=e[o++])t===e[o]&&(i=n.push(o));while(i--)e.splice(n[i],1)}return e},a=at.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=a(e)}else if(3===i||4===i)return e.nodeValue}else for(;t=e[r];r++)n+=a(t);return n},o=at.selectors={cacheLength:50,createPseudo:ut,match:Q,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(rt,it),e[3]=(e[4]||e[5]||"").replace(rt,it),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||at.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&at.error(e[0]),e},PSEUDO:function(e){var n,r=!e[5]&&e[2];return Q.CHILD.test(e[0])?null:(e[3]&&e[4]!==t?e[2]=e[4]:r&&J.test(r)&&(n=bt(r,!0))&&(n=r.indexOf(")",r.length-n)-r.length)&&(e[0]=e[0].slice(0,n),e[2]=r.slice(0,n)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(rt,it).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=N[e+" "];return t||(t=RegExp("(^|"+P+")"+e+"("+P+"|$)"))&&N(e,function(e){return t.test("string"==typeof e.className&&e.className||typeof e.getAttribute!==j&&e.getAttribute("class")||"")})},ATTR:function(e,t,n){return function(r){var i=at.attr(r,e);return null==i?"!="===t:t?(i+="","="===t?i===n:"!="===t?i!==n:"^="===t?n&&0===i.indexOf(n):"*="===t?n&&i.indexOf(n)>-1:"$="===t?n&&i.slice(-n.length)===n:"~="===t?(" "+i+" ").indexOf(n)>-1:"|="===t?i===n||i.slice(0,n.length+1)===n+"-":!1):!0}},CHILD:function(e,t,n,r,i){var o="nth"!==e.slice(0,3),a="last"!==e.slice(-4),s="of-type"===t;return 1===r&&0===i?function(e){return!!e.parentNode}:function(t,n,l){var u,c,p,f,d,h,g=o!==a?"nextSibling":"previousSibling",m=t.parentNode,y=s&&t.nodeName.toLowerCase(),v=!l&&!s;if(m){if(o){while(g){p=t;while(p=p[g])if(s?p.nodeName.toLowerCase()===y:1===p.nodeType)return!1;h=g="only"===e&&!h&&"nextSibling"}return!0}if(h=[a?m.firstChild:m.lastChild],a&&v){c=m[b]||(m[b]={}),u=c[e]||[],d=u[0]===T&&u[1],f=u[0]===T&&u[2],p=d&&m.childNodes[d];while(p=++d&&p&&p[g]||(f=d=0)||h.pop())if(1===p.nodeType&&++f&&p===t){c[e]=[T,d,f];break}}else if(v&&(u=(t[b]||(t[b]={}))[e])&&u[0]===T)f=u[1];else while(p=++d&&p&&p[g]||(f=d=0)||h.pop())if((s?p.nodeName.toLowerCase()===y:1===p.nodeType)&&++f&&(v&&((p[b]||(p[b]={}))[e]=[T,f]),p===t))break;return f-=i,f===r||0===f%r&&f/r>=0}}},PSEUDO:function(e,t){var n,r=o.pseudos[e]||o.setFilters[e.toLowerCase()]||at.error("unsupported pseudo: "+e);return r[b]?r(t):r.length>1?(n=[e,e,"",t],o.setFilters.hasOwnProperty(e.toLowerCase())?ut(function(e,n){var i,o=r(e,t),a=o.length;while(a--)i=F.call(e,o[a]),e[i]=!(n[i]=o[a])}):function(e){return r(e,0,n)}):r}},pseudos:{not:ut(function(e){var t=[],n=[],r=l(e.replace(z,"$1"));return r[b]?ut(function(e,t,n,i){var o,a=r(e,null,i,[]),s=e.length;while(s--)(o=a[s])&&(e[s]=!(t[s]=o))}):function(e,i,o){return t[0]=e,r(t,null,o,n),!n.pop()}}),has:ut(function(e){return function(t){return at(e,t).length>0}}),contains:ut(function(e){return function(t){return(t.textContent||t.innerText||a(t)).indexOf(e)>-1}}),lang:ut(function(e){return G.test(e||"")||at.error("unsupported lang: "+e),e=e.replace(rt,it).toLowerCase(),function(t){var n;do if(n=h?t.lang:t.getAttribute("xml:lang")||t.getAttribute("lang"))return n=n.toLowerCase(),n===e||0===n.indexOf(e+"-");while((t=t.parentNode)&&1===t.nodeType);return!1}}),target:function(t){var n=e.location&&e.location.hash;return n&&n.slice(1)===t.id},root:function(e){return e===d},focus:function(e){return e===f.activeElement&&(!f.hasFocus||f.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:function(e){return e.disabled===!1},disabled:function(e){return e.disabled===!0},checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,e.selected===!0},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeName>"@"||3===e.nodeType||4===e.nodeType)return!1;return!0},parent:function(e){return!o.pseudos.empty(e)},header:function(e){return tt.test(e.nodeName)},input:function(e){return et.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||t.toLowerCase()===e.type)},first:vt(function(){return[0]}),last:vt(function(e,t){return[t-1]}),eq:vt(function(e,t,n){return[0>n?n+t:n]}),even:vt(function(e,t){var n=0;for(;t>n;n+=2)e.push(n);return e}),odd:vt(function(e,t){var n=1;for(;t>n;n+=2)e.push(n);return e}),lt:vt(function(e,t,n){var r=0>n?n+t:n;for(;--r>=0;)e.push(r);return e}),gt:vt(function(e,t,n){var r=0>n?n+t:n;for(;t>++r;)e.push(r);return e})}};for(n in{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})o.pseudos[n]=mt(n);for(n in{submit:!0,reset:!0})o.pseudos[n]=yt(n);function bt(e,t){var n,r,i,a,s,l,u,c=k[e+" "];if(c)return t?0:c.slice(0);s=e,l=[],u=o.preFilter;while(s){(!n||(r=X.exec(s)))&&(r&&(s=s.slice(r[0].length)||s),l.push(i=[])),n=!1,(r=U.exec(s))&&(n=r.shift(),i.push({value:n,type:r[0].replace(z," ")}),s=s.slice(n.length));for(a in o.filter)!(r=Q[a].exec(s))||u[a]&&!(r=u[a](r))||(n=r.shift(),i.push({value:n,type:a,matches:r}),s=s.slice(n.length));if(!n)break}return t?s.length:s?at.error(e):k(e,l).slice(0)}function xt(e){var t=0,n=e.length,r="";for(;n>t;t++)r+=e[t].value;return r}function wt(e,t,n){var r=t.dir,o=n&&"parentNode"===r,a=C++;return t.first?function(t,n,i){while(t=t[r])if(1===t.nodeType||o)return e(t,n,i)}:function(t,n,s){var l,u,c,p=T+" "+a;if(s){while(t=t[r])if((1===t.nodeType||o)&&e(t,n,s))return!0}else while(t=t[r])if(1===t.nodeType||o)if(c=t[b]||(t[b]={}),(u=c[r])&&u[0]===p){if((l=u[1])===!0||l===i)return l===!0}else if(u=c[r]=[p],u[1]=e(t,n,s)||i,u[1]===!0)return!0}}function Tt(e){return e.length>1?function(t,n,r){var i=e.length;while(i--)if(!e[i](t,n,r))return!1;return!0}:e[0]}function Ct(e,t,n,r,i){var o,a=[],s=0,l=e.length,u=null!=t;for(;l>s;s++)(o=e[s])&&(!n||n(o,r,i))&&(a.push(o),u&&t.push(s));return a}function Nt(e,t,n,r,i,o){return r&&!r[b]&&(r=Nt(r)),i&&!i[b]&&(i=Nt(i,o)),ut(function(o,a,s,l){var u,c,p,f=[],d=[],h=a.length,g=o||St(t||"*",s.nodeType?[s]:s,[]),m=!e||!o&&t?g:Ct(g,f,e,s,l),y=n?i||(o?e:h||r)?[]:a:m;if(n&&n(m,y,s,l),r){u=Ct(y,d),r(u,[],s,l),c=u.length;while(c--)(p=u[c])&&(y[d[c]]=!(m[d[c]]=p))}if(o){if(i||e){if(i){u=[],c=y.length;while(c--)(p=y[c])&&u.push(m[c]=p);i(null,y=[],u,l)}c=y.length;while(c--)(p=y[c])&&(u=i?F.call(o,p):f[c])>-1&&(o[u]=!(a[u]=p))}}else y=Ct(y===a?y.splice(h,y.length):y),i?i(null,a,y,l):M.apply(a,y)})}function kt(e){var t,n,r,i=e.length,a=o.relative[e[0].type],s=a||o.relative[" "],l=a?1:0,c=wt(function(e){return e===t},s,!0),p=wt(function(e){return F.call(t,e)>-1},s,!0),f=[function(e,n,r){return!a&&(r||n!==u)||((t=n).nodeType?c(e,n,r):p(e,n,r))}];for(;i>l;l++)if(n=o.relative[e[l].type])f=[wt(Tt(f),n)];else{if(n=o.filter[e[l].type].apply(null,e[l].matches),n[b]){for(r=++l;i>r;r++)if(o.relative[e[r].type])break;return Nt(l>1&&Tt(f),l>1&&xt(e.slice(0,l-1).concat({value:" "===e[l-2].type?"*":""})).replace(z,"$1"),n,r>l&&kt(e.slice(l,r)),i>r&&kt(e=e.slice(r)),i>r&&xt(e))}f.push(n)}return Tt(f)}function Et(e,t){var n=0,r=t.length>0,a=e.length>0,s=function(s,l,c,p,d){var h,g,m,y=[],v=0,b="0",x=s&&[],w=null!=d,C=u,N=s||a&&o.find.TAG("*",d&&l.parentNode||l),k=T+=null==C?1:Math.random()||.1;for(w&&(u=l!==f&&l,i=n);null!=(h=N[b]);b++){if(a&&h){g=0;while(m=e[g++])if(m(h,l,c)){p.push(h);break}w&&(T=k,i=++n)}r&&((h=!m&&h)&&v--,s&&x.push(h))}if(v+=b,r&&b!==v){g=0;while(m=t[g++])m(x,y,l,c);if(s){if(v>0)while(b--)x[b]||y[b]||(y[b]=q.call(p));y=Ct(y)}M.apply(p,y),w&&!s&&y.length>0&&v+t.length>1&&at.uniqueSort(p)}return w&&(T=k,u=C),x};return r?ut(s):s}l=at.compile=function(e,t){var n,r=[],i=[],o=E[e+" "];if(!o){t||(t=bt(e)),n=t.length;while(n--)o=kt(t[n]),o[b]?r.push(o):i.push(o);o=E(e,Et(i,r))}return o};function St(e,t,n){var r=0,i=t.length;for(;i>r;r++)at(e,t[r],n);return n}function At(e,t,n,i){var a,s,u,c,p,f=bt(e);if(!i&&1===f.length){if(s=f[0]=f[0].slice(0),s.length>2&&"ID"===(u=s[0]).type&&r.getById&&9===t.nodeType&&h&&o.relative[s[1].type]){if(t=(o.find.ID(u.matches[0].replace(rt,it),t)||[])[0],!t)return n;e=e.slice(s.shift().value.length)}a=Q.needsContext.test(e)?0:s.length;while(a--){if(u=s[a],o.relative[c=u.type])break;if((p=o.find[c])&&(i=p(u.matches[0].replace(rt,it),V.test(s[0].type)&&t.parentNode||t))){if(s.splice(a,1),e=i.length&&xt(s),!e)return M.apply(n,i),n;break}}}return l(e,f)(i,t,!h,n,V.test(e)),n}o.pseudos.nth=o.pseudos.eq;function jt(){}jt.prototype=o.filters=o.pseudos,o.setFilters=new jt,r.sortStable=b.split("").sort(A).join("")===b,p(),[0,0].sort(A),r.detectDuplicates=S,x.find=at,x.expr=at.selectors,x.expr[":"]=x.expr.pseudos,x.unique=at.uniqueSort,x.text=at.getText,x.isXMLDoc=at.isXML,x.contains=at.contains}(e);var O={};function F(e){var t=O[e]={};return x.each(e.match(T)||[],function(e,n){t[n]=!0}),t}x.Callbacks=function(e){e="string"==typeof e?O[e]||F(e):x.extend({},e);var n,r,i,o,a,s,l=[],u=!e.once&&[],c=function(t){for(r=e.memory&&t,i=!0,a=s||0,s=0,o=l.length,n=!0;l&&o>a;a++)if(l[a].apply(t[0],t[1])===!1&&e.stopOnFalse){r=!1;break}n=!1,l&&(u?u.length&&c(u.shift()):r?l=[]:p.disable())},p={add:function(){if(l){var t=l.length;(function i(t){x.each(t,function(t,n){var r=x.type(n);"function"===r?e.unique&&p.has(n)||l.push(n):n&&n.length&&"string"!==r&&i(n)})})(arguments),n?o=l.length:r&&(s=t,c(r))}return this},remove:function(){return l&&x.each(arguments,function(e,t){var r;while((r=x.inArray(t,l,r))>-1)l.splice(r,1),n&&(o>=r&&o--,a>=r&&a--)}),this},has:function(e){return e?x.inArray(e,l)>-1:!(!l||!l.length)},empty:function(){return l=[],o=0,this},disable:function(){return l=u=r=t,this},disabled:function(){return!l},lock:function(){return u=t,r||p.disable(),this},locked:function(){return!u},fireWith:function(e,t){return t=t||[],t=[e,t.slice?t.slice():t],!l||i&&!u||(n?u.push(t):c(t)),this},fire:function(){return p.fireWith(this,arguments),this},fired:function(){return!!i}};return p},x.extend({Deferred:function(e){var t=[["resolve","done",x.Callbacks("once memory"),"resolved"],["reject","fail",x.Callbacks("once memory"),"rejected"],["notify","progress",x.Callbacks("memory")]],n="pending",r={state:function(){return n},always:function(){return i.done(arguments).fail(arguments),this},then:function(){var e=arguments;return x.Deferred(function(n){x.each(t,function(t,o){var a=o[0],s=x.isFunction(e[t])&&e[t];i[o[1]](function(){var e=s&&s.apply(this,arguments);e&&x.isFunction(e.promise)?e.promise().done(n.resolve).fail(n.reject).progress(n.notify):n[a+"With"](this===r?n.promise():this,s?[e]:arguments)})}),e=null}).promise()},promise:function(e){return null!=e?x.extend(e,r):r}},i={};return r.pipe=r.then,x.each(t,function(e,o){var a=o[2],s=o[3];r[o[1]]=a.add,s&&a.add(function(){n=s},t[1^e][2].disable,t[2][2].lock),i[o[0]]=function(){return i[o[0]+"With"](this===i?r:this,arguments),this},i[o[0]+"With"]=a.fireWith}),r.promise(i),e&&e.call(i,i),i},when:function(e){var t=0,n=g.call(arguments),r=n.length,i=1!==r||e&&x.isFunction(e.promise)?r:0,o=1===i?e:x.Deferred(),a=function(e,t,n){return function(r){t[e]=this,n[e]=arguments.length>1?g.call(arguments):r,n===s?o.notifyWith(t,n):--i||o.resolveWith(t,n)}},s,l,u;if(r>1)for(s=Array(r),l=Array(r),u=Array(r);r>t;t++)n[t]&&x.isFunction(n[t].promise)?n[t].promise().done(a(t,u,n)).fail(o.reject).progress(a(t,l,s)):--i;return i||o.resolveWith(u,n),o.promise()}}),x.support=function(t){var n,r,o,s,l,u,c,p,f,d=a.createElement("div");if(d.setAttribute("className","t"),d.innerHTML="
a",n=d.getElementsByTagName("*")||[],r=d.getElementsByTagName("a")[0],!r||!r.style||!n.length)return t;s=a.createElement("select"),u=s.appendChild(a.createElement("option")),o=d.getElementsByTagName("input")[0],r.style.cssText="top:1px;float:left;opacity:.5",t.getSetAttribute="t"!==d.className,t.leadingWhitespace=3===d.firstChild.nodeType,t.tbody=!d.getElementsByTagName("tbody").length,t.htmlSerialize=!!d.getElementsByTagName("link").length,t.style=/top/.test(r.getAttribute("style")),t.hrefNormalized="/a"===r.getAttribute("href"),t.opacity=/^0.5/.test(r.style.opacity),t.cssFloat=!!r.style.cssFloat,t.checkOn=!!o.value,t.optSelected=u.selected,t.enctype=!!a.createElement("form").enctype,t.html5Clone="<:nav>"!==a.createElement("nav").cloneNode(!0).outerHTML,t.inlineBlockNeedsLayout=!1,t.shrinkWrapBlocks=!1,t.pixelPosition=!1,t.deleteExpando=!0,t.noCloneEvent=!0,t.reliableMarginRight=!0,t.boxSizingReliable=!0,o.checked=!0,t.noCloneChecked=o.cloneNode(!0).checked,s.disabled=!0,t.optDisabled=!u.disabled;try{delete d.test}catch(h){t.deleteExpando=!1}o=a.createElement("input"),o.setAttribute("value",""),t.input=""===o.getAttribute("value"),o.value="t",o.setAttribute("type","radio"),t.radioValue="t"===o.value,o.setAttribute("checked","t"),o.setAttribute("name","t"),l=a.createDocumentFragment(),l.appendChild(o),t.appendChecked=o.checked,t.checkClone=l.cloneNode(!0).cloneNode(!0).lastChild.checked,d.attachEvent&&(d.attachEvent("onclick",function(){t.noCloneEvent=!1}),d.cloneNode(!0).click());for(f in{submit:!0,change:!0,focusin:!0})d.setAttribute(c="on"+f,"t"),t[f+"Bubbles"]=c in e||d.attributes[c].expando===!1;d.style.backgroundClip="content-box",d.cloneNode(!0).style.backgroundClip="",t.clearCloneStyle="content-box"===d.style.backgroundClip;for(f in x(t))break;return t.ownLast="0"!==f,x(function(){var n,r,o,s="padding:0;margin:0;border:0;display:block;box-sizing:content-box;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;",l=a.getElementsByTagName("body")[0];l&&(n=a.createElement("div"),n.style.cssText="border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px",l.appendChild(n).appendChild(d),d.innerHTML="
t
",o=d.getElementsByTagName("td"),o[0].style.cssText="padding:0;margin:0;border:0;display:none",p=0===o[0].offsetHeight,o[0].style.display="",o[1].style.display="none",t.reliableHiddenOffsets=p&&0===o[0].offsetHeight,d.innerHTML="",d.style.cssText="box-sizing:border-box;-moz-box-sizing:border-box;-webkit-box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%;",x.swap(l,null!=l.style.zoom?{zoom:1}:{},function(){t.boxSizing=4===d.offsetWidth}),e.getComputedStyle&&(t.pixelPosition="1%"!==(e.getComputedStyle(d,null)||{}).top,t.boxSizingReliable="4px"===(e.getComputedStyle(d,null)||{width:"4px"}).width,r=d.appendChild(a.createElement("div")),r.style.cssText=d.style.cssText=s,r.style.marginRight=r.style.width="0",d.style.width="1px",t.reliableMarginRight=!parseFloat((e.getComputedStyle(r,null)||{}).marginRight)),typeof d.style.zoom!==i&&(d.innerHTML="",d.style.cssText=s+"width:1px;padding:1px;display:inline;zoom:1",t.inlineBlockNeedsLayout=3===d.offsetWidth,d.style.display="block",d.innerHTML="
",d.firstChild.style.width="5px",t.shrinkWrapBlocks=3!==d.offsetWidth,t.inlineBlockNeedsLayout&&(l.style.zoom=1)),l.removeChild(n),n=d=o=r=null)}),n=s=l=u=r=o=null,t}({});var B=/(?:\{[\s\S]*\}|\[[\s\S]*\])$/,P=/([A-Z])/g;function R(e,n,r,i){if(x.acceptData(e)){var o,a,s=x.expando,l=e.nodeType,u=l?x.cache:e,c=l?e[s]:e[s]&&s; -if(c&&u[c]&&(i||u[c].data)||r!==t||"string"!=typeof n)return c||(c=l?e[s]=p.pop()||x.guid++:s),u[c]||(u[c]=l?{}:{toJSON:x.noop}),("object"==typeof n||"function"==typeof n)&&(i?u[c]=x.extend(u[c],n):u[c].data=x.extend(u[c].data,n)),a=u[c],i||(a.data||(a.data={}),a=a.data),r!==t&&(a[x.camelCase(n)]=r),"string"==typeof n?(o=a[n],null==o&&(o=a[x.camelCase(n)])):o=a,o}}function W(e,t,n){if(x.acceptData(e)){var r,i,o=e.nodeType,a=o?x.cache:e,s=o?e[x.expando]:x.expando;if(a[s]){if(t&&(r=n?a[s]:a[s].data)){x.isArray(t)?t=t.concat(x.map(t,x.camelCase)):t in r?t=[t]:(t=x.camelCase(t),t=t in r?[t]:t.split(" ")),i=t.length;while(i--)delete r[t[i]];if(n?!I(r):!x.isEmptyObject(r))return}(n||(delete a[s].data,I(a[s])))&&(o?x.cleanData([e],!0):x.support.deleteExpando||a!=a.window?delete a[s]:a[s]=null)}}}x.extend({cache:{},noData:{applet:!0,embed:!0,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(e){return e=e.nodeType?x.cache[e[x.expando]]:e[x.expando],!!e&&!I(e)},data:function(e,t,n){return R(e,t,n)},removeData:function(e,t){return W(e,t)},_data:function(e,t,n){return R(e,t,n,!0)},_removeData:function(e,t){return W(e,t,!0)},acceptData:function(e){if(e.nodeType&&1!==e.nodeType&&9!==e.nodeType)return!1;var t=e.nodeName&&x.noData[e.nodeName.toLowerCase()];return!t||t!==!0&&e.getAttribute("classid")===t}}),x.fn.extend({data:function(e,n){var r,i,o=null,a=0,s=this[0];if(e===t){if(this.length&&(o=x.data(s),1===s.nodeType&&!x._data(s,"parsedAttrs"))){for(r=s.attributes;r.length>a;a++)i=r[a].name,0===i.indexOf("data-")&&(i=x.camelCase(i.slice(5)),$(s,i,o[i]));x._data(s,"parsedAttrs",!0)}return o}return"object"==typeof e?this.each(function(){x.data(this,e)}):arguments.length>1?this.each(function(){x.data(this,e,n)}):s?$(s,e,x.data(s,e)):null},removeData:function(e){return this.each(function(){x.removeData(this,e)})}});function $(e,n,r){if(r===t&&1===e.nodeType){var i="data-"+n.replace(P,"-$1").toLowerCase();if(r=e.getAttribute(i),"string"==typeof r){try{r="true"===r?!0:"false"===r?!1:"null"===r?null:+r+""===r?+r:B.test(r)?x.parseJSON(r):r}catch(o){}x.data(e,n,r)}else r=t}return r}function I(e){var t;for(t in e)if(("data"!==t||!x.isEmptyObject(e[t]))&&"toJSON"!==t)return!1;return!0}x.extend({queue:function(e,n,r){var i;return e?(n=(n||"fx")+"queue",i=x._data(e,n),r&&(!i||x.isArray(r)?i=x._data(e,n,x.makeArray(r)):i.push(r)),i||[]):t},dequeue:function(e,t){t=t||"fx";var n=x.queue(e,t),r=n.length,i=n.shift(),o=x._queueHooks(e,t),a=function(){x.dequeue(e,t)};"inprogress"===i&&(i=n.shift(),r--),o.cur=i,i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,a,o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return x._data(e,n)||x._data(e,n,{empty:x.Callbacks("once memory").add(function(){x._removeData(e,t+"queue"),x._removeData(e,n)})})}}),x.fn.extend({queue:function(e,n){var r=2;return"string"!=typeof e&&(n=e,e="fx",r--),r>arguments.length?x.queue(this[0],e):n===t?this:this.each(function(){var t=x.queue(this,e,n);x._queueHooks(this,e),"fx"===e&&"inprogress"!==t[0]&&x.dequeue(this,e)})},dequeue:function(e){return this.each(function(){x.dequeue(this,e)})},delay:function(e,t){return e=x.fx?x.fx.speeds[e]||e:e,t=t||"fx",this.queue(t,function(t,n){var r=setTimeout(t,e);n.stop=function(){clearTimeout(r)}})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,n){var r,i=1,o=x.Deferred(),a=this,s=this.length,l=function(){--i||o.resolveWith(a,[a])};"string"!=typeof e&&(n=e,e=t),e=e||"fx";while(s--)r=x._data(a[s],e+"queueHooks"),r&&r.empty&&(i++,r.empty.add(l));return l(),o.promise(n)}});var z,X,U=/[\t\r\n\f]/g,V=/\r/g,Y=/^(?:input|select|textarea|button|object)$/i,J=/^(?:a|area)$/i,G=/^(?:checked|selected)$/i,Q=x.support.getSetAttribute,K=x.support.input;x.fn.extend({attr:function(e,t){return x.access(this,x.attr,e,t,arguments.length>1)},removeAttr:function(e){return this.each(function(){x.removeAttr(this,e)})},prop:function(e,t){return x.access(this,x.prop,e,t,arguments.length>1)},removeProp:function(e){return e=x.propFix[e]||e,this.each(function(){try{this[e]=t,delete this[e]}catch(n){}})},addClass:function(e){var t,n,r,i,o,a=0,s=this.length,l="string"==typeof e&&e;if(x.isFunction(e))return this.each(function(t){x(this).addClass(e.call(this,t,this.className))});if(l)for(t=(e||"").match(T)||[];s>a;a++)if(n=this[a],r=1===n.nodeType&&(n.className?(" "+n.className+" ").replace(U," "):" ")){o=0;while(i=t[o++])0>r.indexOf(" "+i+" ")&&(r+=i+" ");n.className=x.trim(r)}return this},removeClass:function(e){var t,n,r,i,o,a=0,s=this.length,l=0===arguments.length||"string"==typeof e&&e;if(x.isFunction(e))return this.each(function(t){x(this).removeClass(e.call(this,t,this.className))});if(l)for(t=(e||"").match(T)||[];s>a;a++)if(n=this[a],r=1===n.nodeType&&(n.className?(" "+n.className+" ").replace(U," "):"")){o=0;while(i=t[o++])while(r.indexOf(" "+i+" ")>=0)r=r.replace(" "+i+" "," ");n.className=e?x.trim(r):""}return this},toggleClass:function(e,t){var n=typeof e,r="boolean"==typeof t;return x.isFunction(e)?this.each(function(n){x(this).toggleClass(e.call(this,n,this.className,t),t)}):this.each(function(){if("string"===n){var o,a=0,s=x(this),l=t,u=e.match(T)||[];while(o=u[a++])l=r?l:!s.hasClass(o),s[l?"addClass":"removeClass"](o)}else(n===i||"boolean"===n)&&(this.className&&x._data(this,"__className__",this.className),this.className=this.className||e===!1?"":x._data(this,"__className__")||"")})},hasClass:function(e){var t=" "+e+" ",n=0,r=this.length;for(;r>n;n++)if(1===this[n].nodeType&&(" "+this[n].className+" ").replace(U," ").indexOf(t)>=0)return!0;return!1},val:function(e){var n,r,i,o=this[0];{if(arguments.length)return i=x.isFunction(e),this.each(function(n){var o;1===this.nodeType&&(o=i?e.call(this,n,x(this).val()):e,null==o?o="":"number"==typeof o?o+="":x.isArray(o)&&(o=x.map(o,function(e){return null==e?"":e+""})),r=x.valHooks[this.type]||x.valHooks[this.nodeName.toLowerCase()],r&&"set"in r&&r.set(this,o,"value")!==t||(this.value=o))});if(o)return r=x.valHooks[o.type]||x.valHooks[o.nodeName.toLowerCase()],r&&"get"in r&&(n=r.get(o,"value"))!==t?n:(n=o.value,"string"==typeof n?n.replace(V,""):null==n?"":n)}}}),x.extend({valHooks:{option:{get:function(e){var t=x.find.attr(e,"value");return null!=t?t:e.text}},select:{get:function(e){var t,n,r=e.options,i=e.selectedIndex,o="select-one"===e.type||0>i,a=o?null:[],s=o?i+1:r.length,l=0>i?s:o?i:0;for(;s>l;l++)if(n=r[l],!(!n.selected&&l!==i||(x.support.optDisabled?n.disabled:null!==n.getAttribute("disabled"))||n.parentNode.disabled&&x.nodeName(n.parentNode,"optgroup"))){if(t=x(n).val(),o)return t;a.push(t)}return a},set:function(e,t){var n,r,i=e.options,o=x.makeArray(t),a=i.length;while(a--)r=i[a],(r.selected=x.inArray(x(r).val(),o)>=0)&&(n=!0);return n||(e.selectedIndex=-1),o}}},attr:function(e,n,r){var o,a,s=e.nodeType;if(e&&3!==s&&8!==s&&2!==s)return typeof e.getAttribute===i?x.prop(e,n,r):(1===s&&x.isXMLDoc(e)||(n=n.toLowerCase(),o=x.attrHooks[n]||(x.expr.match.bool.test(n)?X:z)),r===t?o&&"get"in o&&null!==(a=o.get(e,n))?a:(a=x.find.attr(e,n),null==a?t:a):null!==r?o&&"set"in o&&(a=o.set(e,r,n))!==t?a:(e.setAttribute(n,r+""),r):(x.removeAttr(e,n),t))},removeAttr:function(e,t){var n,r,i=0,o=t&&t.match(T);if(o&&1===e.nodeType)while(n=o[i++])r=x.propFix[n]||n,x.expr.match.bool.test(n)?K&&Q||!G.test(n)?e[r]=!1:e[x.camelCase("default-"+n)]=e[r]=!1:x.attr(e,n,""),e.removeAttribute(Q?n:r)},attrHooks:{type:{set:function(e,t){if(!x.support.radioValue&&"radio"===t&&x.nodeName(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},propFix:{"for":"htmlFor","class":"className"},prop:function(e,n,r){var i,o,a,s=e.nodeType;if(e&&3!==s&&8!==s&&2!==s)return a=1!==s||!x.isXMLDoc(e),a&&(n=x.propFix[n]||n,o=x.propHooks[n]),r!==t?o&&"set"in o&&(i=o.set(e,r,n))!==t?i:e[n]=r:o&&"get"in o&&null!==(i=o.get(e,n))?i:e[n]},propHooks:{tabIndex:{get:function(e){var t=x.find.attr(e,"tabindex");return t?parseInt(t,10):Y.test(e.nodeName)||J.test(e.nodeName)&&e.href?0:-1}}}}),X={set:function(e,t,n){return t===!1?x.removeAttr(e,n):K&&Q||!G.test(n)?e.setAttribute(!Q&&x.propFix[n]||n,n):e[x.camelCase("default-"+n)]=e[n]=!0,n}},x.each(x.expr.match.bool.source.match(/\w+/g),function(e,n){var r=x.expr.attrHandle[n]||x.find.attr;x.expr.attrHandle[n]=K&&Q||!G.test(n)?function(e,n,i){var o=x.expr.attrHandle[n],a=i?t:(x.expr.attrHandle[n]=t)!=r(e,n,i)?n.toLowerCase():null;return x.expr.attrHandle[n]=o,a}:function(e,n,r){return r?t:e[x.camelCase("default-"+n)]?n.toLowerCase():null}}),K&&Q||(x.attrHooks.value={set:function(e,n,r){return x.nodeName(e,"input")?(e.defaultValue=n,t):z&&z.set(e,n,r)}}),Q||(z={set:function(e,n,r){var i=e.getAttributeNode(r);return i||e.setAttributeNode(i=e.ownerDocument.createAttribute(r)),i.value=n+="","value"===r||n===e.getAttribute(r)?n:t}},x.expr.attrHandle.id=x.expr.attrHandle.name=x.expr.attrHandle.coords=function(e,n,r){var i;return r?t:(i=e.getAttributeNode(n))&&""!==i.value?i.value:null},x.valHooks.button={get:function(e,n){var r=e.getAttributeNode(n);return r&&r.specified?r.value:t},set:z.set},x.attrHooks.contenteditable={set:function(e,t,n){z.set(e,""===t?!1:t,n)}},x.each(["width","height"],function(e,n){x.attrHooks[n]={set:function(e,r){return""===r?(e.setAttribute(n,"auto"),r):t}}})),x.support.hrefNormalized||x.each(["href","src"],function(e,t){x.propHooks[t]={get:function(e){return e.getAttribute(t,4)}}}),x.support.style||(x.attrHooks.style={get:function(e){return e.style.cssText||t},set:function(e,t){return e.style.cssText=t+""}}),x.support.optSelected||(x.propHooks.selected={get:function(e){var t=e.parentNode;return t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex),null}}),x.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){x.propFix[this.toLowerCase()]=this}),x.support.enctype||(x.propFix.enctype="encoding"),x.each(["radio","checkbox"],function(){x.valHooks[this]={set:function(e,n){return x.isArray(n)?e.checked=x.inArray(x(e).val(),n)>=0:t}},x.support.checkOn||(x.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})});var Z=/^(?:input|select|textarea)$/i,et=/^key/,tt=/^(?:mouse|contextmenu)|click/,nt=/^(?:focusinfocus|focusoutblur)$/,rt=/^([^.]*)(?:\.(.+)|)$/;function it(){return!0}function ot(){return!1}function at(){try{return a.activeElement}catch(e){}}x.event={global:{},add:function(e,n,r,o,a){var s,l,u,c,p,f,d,h,g,m,y,v=x._data(e);if(v){r.handler&&(c=r,r=c.handler,a=c.selector),r.guid||(r.guid=x.guid++),(l=v.events)||(l=v.events={}),(f=v.handle)||(f=v.handle=function(e){return typeof x===i||e&&x.event.triggered===e.type?t:x.event.dispatch.apply(f.elem,arguments)},f.elem=e),n=(n||"").match(T)||[""],u=n.length;while(u--)s=rt.exec(n[u])||[],g=y=s[1],m=(s[2]||"").split(".").sort(),g&&(p=x.event.special[g]||{},g=(a?p.delegateType:p.bindType)||g,p=x.event.special[g]||{},d=x.extend({type:g,origType:y,data:o,handler:r,guid:r.guid,selector:a,needsContext:a&&x.expr.match.needsContext.test(a),namespace:m.join(".")},c),(h=l[g])||(h=l[g]=[],h.delegateCount=0,p.setup&&p.setup.call(e,o,m,f)!==!1||(e.addEventListener?e.addEventListener(g,f,!1):e.attachEvent&&e.attachEvent("on"+g,f))),p.add&&(p.add.call(e,d),d.handler.guid||(d.handler.guid=r.guid)),a?h.splice(h.delegateCount++,0,d):h.push(d),x.event.global[g]=!0);e=null}},remove:function(e,t,n,r,i){var o,a,s,l,u,c,p,f,d,h,g,m=x.hasData(e)&&x._data(e);if(m&&(c=m.events)){t=(t||"").match(T)||[""],u=t.length;while(u--)if(s=rt.exec(t[u])||[],d=g=s[1],h=(s[2]||"").split(".").sort(),d){p=x.event.special[d]||{},d=(r?p.delegateType:p.bindType)||d,f=c[d]||[],s=s[2]&&RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),l=o=f.length;while(o--)a=f[o],!i&&g!==a.origType||n&&n.guid!==a.guid||s&&!s.test(a.namespace)||r&&r!==a.selector&&("**"!==r||!a.selector)||(f.splice(o,1),a.selector&&f.delegateCount--,p.remove&&p.remove.call(e,a));l&&!f.length&&(p.teardown&&p.teardown.call(e,h,m.handle)!==!1||x.removeEvent(e,d,m.handle),delete c[d])}else for(d in c)x.event.remove(e,d+t[u],n,r,!0);x.isEmptyObject(c)&&(delete m.handle,x._removeData(e,"events"))}},trigger:function(n,r,i,o){var s,l,u,c,p,f,d,h=[i||a],g=v.call(n,"type")?n.type:n,m=v.call(n,"namespace")?n.namespace.split("."):[];if(u=f=i=i||a,3!==i.nodeType&&8!==i.nodeType&&!nt.test(g+x.event.triggered)&&(g.indexOf(".")>=0&&(m=g.split("."),g=m.shift(),m.sort()),l=0>g.indexOf(":")&&"on"+g,n=n[x.expando]?n:new x.Event(g,"object"==typeof n&&n),n.isTrigger=o?2:3,n.namespace=m.join("."),n.namespace_re=n.namespace?RegExp("(^|\\.)"+m.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,n.result=t,n.target||(n.target=i),r=null==r?[n]:x.makeArray(r,[n]),p=x.event.special[g]||{},o||!p.trigger||p.trigger.apply(i,r)!==!1)){if(!o&&!p.noBubble&&!x.isWindow(i)){for(c=p.delegateType||g,nt.test(c+g)||(u=u.parentNode);u;u=u.parentNode)h.push(u),f=u;f===(i.ownerDocument||a)&&h.push(f.defaultView||f.parentWindow||e)}d=0;while((u=h[d++])&&!n.isPropagationStopped())n.type=d>1?c:p.bindType||g,s=(x._data(u,"events")||{})[n.type]&&x._data(u,"handle"),s&&s.apply(u,r),s=l&&u[l],s&&x.acceptData(u)&&s.apply&&s.apply(u,r)===!1&&n.preventDefault();if(n.type=g,!o&&!n.isDefaultPrevented()&&(!p._default||p._default.apply(h.pop(),r)===!1)&&x.acceptData(i)&&l&&i[g]&&!x.isWindow(i)){f=i[l],f&&(i[l]=null),x.event.triggered=g;try{i[g]()}catch(y){}x.event.triggered=t,f&&(i[l]=f)}return n.result}},dispatch:function(e){e=x.event.fix(e);var n,r,i,o,a,s=[],l=g.call(arguments),u=(x._data(this,"events")||{})[e.type]||[],c=x.event.special[e.type]||{};if(l[0]=e,e.delegateTarget=this,!c.preDispatch||c.preDispatch.call(this,e)!==!1){s=x.event.handlers.call(this,e,u),n=0;while((o=s[n++])&&!e.isPropagationStopped()){e.currentTarget=o.elem,a=0;while((i=o.handlers[a++])&&!e.isImmediatePropagationStopped())(!e.namespace_re||e.namespace_re.test(i.namespace))&&(e.handleObj=i,e.data=i.data,r=((x.event.special[i.origType]||{}).handle||i.handler).apply(o.elem,l),r!==t&&(e.result=r)===!1&&(e.preventDefault(),e.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,e),e.result}},handlers:function(e,n){var r,i,o,a,s=[],l=n.delegateCount,u=e.target;if(l&&u.nodeType&&(!e.button||"click"!==e.type))for(;u!=this;u=u.parentNode||this)if(1===u.nodeType&&(u.disabled!==!0||"click"!==e.type)){for(o=[],a=0;l>a;a++)i=n[a],r=i.selector+" ",o[r]===t&&(o[r]=i.needsContext?x(r,this).index(u)>=0:x.find(r,this,null,[u]).length),o[r]&&o.push(i);o.length&&s.push({elem:u,handlers:o})}return n.length>l&&s.push({elem:this,handlers:n.slice(l)}),s},fix:function(e){if(e[x.expando])return e;var t,n,r,i=e.type,o=e,s=this.fixHooks[i];s||(this.fixHooks[i]=s=tt.test(i)?this.mouseHooks:et.test(i)?this.keyHooks:{}),r=s.props?this.props.concat(s.props):this.props,e=new x.Event(o),t=r.length;while(t--)n=r[t],e[n]=o[n];return e.target||(e.target=o.srcElement||a),3===e.target.nodeType&&(e.target=e.target.parentNode),e.metaKey=!!e.metaKey,s.filter?s.filter(e,o):e},props:"altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "),fixHooks:{},keyHooks:{props:"char charCode key keyCode".split(" "),filter:function(e,t){return null==e.which&&(e.which=null!=t.charCode?t.charCode:t.keyCode),e}},mouseHooks:{props:"button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "),filter:function(e,n){var r,i,o,s=n.button,l=n.fromElement;return null==e.pageX&&null!=n.clientX&&(i=e.target.ownerDocument||a,o=i.documentElement,r=i.body,e.pageX=n.clientX+(o&&o.scrollLeft||r&&r.scrollLeft||0)-(o&&o.clientLeft||r&&r.clientLeft||0),e.pageY=n.clientY+(o&&o.scrollTop||r&&r.scrollTop||0)-(o&&o.clientTop||r&&r.clientTop||0)),!e.relatedTarget&&l&&(e.relatedTarget=l===e.target?n.toElement:l),e.which||s===t||(e.which=1&s?1:2&s?3:4&s?2:0),e}},special:{load:{noBubble:!0},focus:{trigger:function(){if(this!==at()&&this.focus)try{return this.focus(),!1}catch(e){}},delegateType:"focusin"},blur:{trigger:function(){return this===at()&&this.blur?(this.blur(),!1):t},delegateType:"focusout"},click:{trigger:function(){return x.nodeName(this,"input")&&"checkbox"===this.type&&this.click?(this.click(),!1):t},_default:function(e){return x.nodeName(e.target,"a")}},beforeunload:{postDispatch:function(e){e.result!==t&&(e.originalEvent.returnValue=e.result)}}},simulate:function(e,t,n,r){var i=x.extend(new x.Event,n,{type:e,isSimulated:!0,originalEvent:{}});r?x.event.trigger(i,null,t):x.event.dispatch.call(t,i),i.isDefaultPrevented()&&n.preventDefault()}},x.removeEvent=a.removeEventListener?function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n,!1)}:function(e,t,n){var r="on"+t;e.detachEvent&&(typeof e[r]===i&&(e[r]=null),e.detachEvent(r,n))},x.Event=function(e,n){return this instanceof x.Event?(e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||e.returnValue===!1||e.getPreventDefault&&e.getPreventDefault()?it:ot):this.type=e,n&&x.extend(this,n),this.timeStamp=e&&e.timeStamp||x.now(),this[x.expando]=!0,t):new x.Event(e,n)},x.Event.prototype={isDefaultPrevented:ot,isPropagationStopped:ot,isImmediatePropagationStopped:ot,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=it,e&&(e.preventDefault?e.preventDefault():e.returnValue=!1)},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=it,e&&(e.stopPropagation&&e.stopPropagation(),e.cancelBubble=!0)},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=it,this.stopPropagation()}},x.each({mouseenter:"mouseover",mouseleave:"mouseout"},function(e,t){x.event.special[e]={delegateType:t,bindType:t,handle:function(e){var n,r=this,i=e.relatedTarget,o=e.handleObj;return(!i||i!==r&&!x.contains(r,i))&&(e.type=o.origType,n=o.handler.apply(this,arguments),e.type=t),n}}}),x.support.submitBubbles||(x.event.special.submit={setup:function(){return x.nodeName(this,"form")?!1:(x.event.add(this,"click._submit keypress._submit",function(e){var n=e.target,r=x.nodeName(n,"input")||x.nodeName(n,"button")?n.form:t;r&&!x._data(r,"submitBubbles")&&(x.event.add(r,"submit._submit",function(e){e._submit_bubble=!0}),x._data(r,"submitBubbles",!0))}),t)},postDispatch:function(e){e._submit_bubble&&(delete e._submit_bubble,this.parentNode&&!e.isTrigger&&x.event.simulate("submit",this.parentNode,e,!0))},teardown:function(){return x.nodeName(this,"form")?!1:(x.event.remove(this,"._submit"),t)}}),x.support.changeBubbles||(x.event.special.change={setup:function(){return Z.test(this.nodeName)?(("checkbox"===this.type||"radio"===this.type)&&(x.event.add(this,"propertychange._change",function(e){"checked"===e.originalEvent.propertyName&&(this._just_changed=!0)}),x.event.add(this,"click._change",function(e){this._just_changed&&!e.isTrigger&&(this._just_changed=!1),x.event.simulate("change",this,e,!0)})),!1):(x.event.add(this,"beforeactivate._change",function(e){var t=e.target;Z.test(t.nodeName)&&!x._data(t,"changeBubbles")&&(x.event.add(t,"change._change",function(e){!this.parentNode||e.isSimulated||e.isTrigger||x.event.simulate("change",this.parentNode,e,!0)}),x._data(t,"changeBubbles",!0))}),t)},handle:function(e){var n=e.target;return this!==n||e.isSimulated||e.isTrigger||"radio"!==n.type&&"checkbox"!==n.type?e.handleObj.handler.apply(this,arguments):t},teardown:function(){return x.event.remove(this,"._change"),!Z.test(this.nodeName)}}),x.support.focusinBubbles||x.each({focus:"focusin",blur:"focusout"},function(e,t){var n=0,r=function(e){x.event.simulate(t,e.target,x.event.fix(e),!0)};x.event.special[t]={setup:function(){0===n++&&a.addEventListener(e,r,!0)},teardown:function(){0===--n&&a.removeEventListener(e,r,!0)}}}),x.fn.extend({on:function(e,n,r,i,o){var a,s;if("object"==typeof e){"string"!=typeof n&&(r=r||n,n=t);for(a in e)this.on(a,n,r,e[a],o);return this}if(null==r&&null==i?(i=n,r=n=t):null==i&&("string"==typeof n?(i=r,r=t):(i=r,r=n,n=t)),i===!1)i=ot;else if(!i)return this;return 1===o&&(s=i,i=function(e){return x().off(e),s.apply(this,arguments)},i.guid=s.guid||(s.guid=x.guid++)),this.each(function(){x.event.add(this,e,i,r,n)})},one:function(e,t,n,r){return this.on(e,t,n,r,1)},off:function(e,n,r){var i,o;if(e&&e.preventDefault&&e.handleObj)return i=e.handleObj,x(e.delegateTarget).off(i.namespace?i.origType+"."+i.namespace:i.origType,i.selector,i.handler),this;if("object"==typeof e){for(o in e)this.off(o,n,e[o]);return this}return(n===!1||"function"==typeof n)&&(r=n,n=t),r===!1&&(r=ot),this.each(function(){x.event.remove(this,e,r,n)})},trigger:function(e,t){return this.each(function(){x.event.trigger(e,t,this)})},triggerHandler:function(e,n){var r=this[0];return r?x.event.trigger(e,n,r,!0):t}});var st=/^.[^:#\[\.,]*$/,lt=/^(?:parents|prev(?:Until|All))/,ut=x.expr.match.needsContext,ct={children:!0,contents:!0,next:!0,prev:!0};x.fn.extend({find:function(e){var t,n=[],r=this,i=r.length;if("string"!=typeof e)return this.pushStack(x(e).filter(function(){for(t=0;i>t;t++)if(x.contains(r[t],this))return!0}));for(t=0;i>t;t++)x.find(e,r[t],n);return n=this.pushStack(i>1?x.unique(n):n),n.selector=this.selector?this.selector+" "+e:e,n},has:function(e){var t,n=x(e,this),r=n.length;return this.filter(function(){for(t=0;r>t;t++)if(x.contains(this,n[t]))return!0})},not:function(e){return this.pushStack(ft(this,e||[],!0))},filter:function(e){return this.pushStack(ft(this,e||[],!1))},is:function(e){return!!ft(this,"string"==typeof e&&ut.test(e)?x(e):e||[],!1).length},closest:function(e,t){var n,r=0,i=this.length,o=[],a=ut.test(e)||"string"!=typeof e?x(e,t||this.context):0;for(;i>r;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(11>n.nodeType&&(a?a.index(n)>-1:1===n.nodeType&&x.find.matchesSelector(n,e))){n=o.push(n);break}return this.pushStack(o.length>1?x.unique(o):o)},index:function(e){return e?"string"==typeof e?x.inArray(this[0],x(e)):x.inArray(e.jquery?e[0]:e,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){var n="string"==typeof e?x(e,t):x.makeArray(e&&e.nodeType?[e]:e),r=x.merge(this.get(),n);return this.pushStack(x.unique(r))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}});function pt(e,t){do e=e[t];while(e&&1!==e.nodeType);return e}x.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return x.dir(e,"parentNode")},parentsUntil:function(e,t,n){return x.dir(e,"parentNode",n)},next:function(e){return pt(e,"nextSibling")},prev:function(e){return pt(e,"previousSibling")},nextAll:function(e){return x.dir(e,"nextSibling")},prevAll:function(e){return x.dir(e,"previousSibling")},nextUntil:function(e,t,n){return x.dir(e,"nextSibling",n)},prevUntil:function(e,t,n){return x.dir(e,"previousSibling",n)},siblings:function(e){return x.sibling((e.parentNode||{}).firstChild,e)},children:function(e){return x.sibling(e.firstChild)},contents:function(e){return x.nodeName(e,"iframe")?e.contentDocument||e.contentWindow.document:x.merge([],e.childNodes)}},function(e,t){x.fn[e]=function(n,r){var i=x.map(this,t,n);return"Until"!==e.slice(-5)&&(r=n),r&&"string"==typeof r&&(i=x.filter(r,i)),this.length>1&&(ct[e]||(i=x.unique(i)),lt.test(e)&&(i=i.reverse())),this.pushStack(i)}}),x.extend({filter:function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?x.find.matchesSelector(r,e)?[r]:[]:x.find.matches(e,x.grep(t,function(e){return 1===e.nodeType}))},dir:function(e,n,r){var i=[],o=e[n];while(o&&9!==o.nodeType&&(r===t||1!==o.nodeType||!x(o).is(r)))1===o.nodeType&&i.push(o),o=o[n];return i},sibling:function(e,t){var n=[];for(;e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n}});function ft(e,t,n){if(x.isFunction(t))return x.grep(e,function(e,r){return!!t.call(e,r,e)!==n});if(t.nodeType)return x.grep(e,function(e){return e===t!==n});if("string"==typeof t){if(st.test(t))return x.filter(t,e,n);t=x.filter(t,e)}return x.grep(e,function(e){return x.inArray(e,t)>=0!==n})}function dt(e){var t=ht.split("|"),n=e.createDocumentFragment();if(n.createElement)while(t.length)n.createElement(t.pop());return n}var ht="abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",gt=/ jQuery\d+="(?:null|\d+)"/g,mt=RegExp("<(?:"+ht+")[\\s/>]","i"),yt=/^\s+/,vt=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,bt=/<([\w:]+)/,xt=/\s*$/g,At={option:[1,""],legend:[1,"
","
"],area:[1,"",""],param:[1,"",""],thead:[1,"","
"],tr:[2,"","
"],col:[2,"","
"],td:[3,"","
"],_default:x.support.htmlSerialize?[0,"",""]:[1,"X
","
"]},jt=dt(a),Dt=jt.appendChild(a.createElement("div"));At.optgroup=At.option,At.tbody=At.tfoot=At.colgroup=At.caption=At.thead,At.th=At.td,x.fn.extend({text:function(e){return x.access(this,function(e){return e===t?x.text(this):this.empty().append((this[0]&&this[0].ownerDocument||a).createTextNode(e))},null,e,arguments.length)},append:function(){return this.domManip(arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Lt(this,e);t.appendChild(e)}})},prepend:function(){return this.domManip(arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Lt(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return this.domManip(arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return this.domManip(arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},remove:function(e,t){var n,r=e?x.filter(e,this):this,i=0;for(;null!=(n=r[i]);i++)t||1!==n.nodeType||x.cleanData(Ft(n)),n.parentNode&&(t&&x.contains(n.ownerDocument,n)&&_t(Ft(n,"script")),n.parentNode.removeChild(n));return this},empty:function(){var e,t=0;for(;null!=(e=this[t]);t++){1===e.nodeType&&x.cleanData(Ft(e,!1));while(e.firstChild)e.removeChild(e.firstChild);e.options&&x.nodeName(e,"select")&&(e.options.length=0)}return this},clone:function(e,t){return e=null==e?!1:e,t=null==t?e:t,this.map(function(){return x.clone(this,e,t)})},html:function(e){return x.access(this,function(e){var n=this[0]||{},r=0,i=this.length;if(e===t)return 1===n.nodeType?n.innerHTML.replace(gt,""):t;if(!("string"!=typeof e||Tt.test(e)||!x.support.htmlSerialize&&mt.test(e)||!x.support.leadingWhitespace&&yt.test(e)||At[(bt.exec(e)||["",""])[1].toLowerCase()])){e=e.replace(vt,"<$1>");try{for(;i>r;r++)n=this[r]||{},1===n.nodeType&&(x.cleanData(Ft(n,!1)),n.innerHTML=e);n=0}catch(o){}}n&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var e=x.map(this,function(e){return[e.nextSibling,e.parentNode]}),t=0;return this.domManip(arguments,function(n){var r=e[t++],i=e[t++];i&&(r&&r.parentNode!==i&&(r=this.nextSibling),x(this).remove(),i.insertBefore(n,r))},!0),t?this:this.remove()},detach:function(e){return this.remove(e,!0)},domManip:function(e,t,n){e=d.apply([],e);var r,i,o,a,s,l,u=0,c=this.length,p=this,f=c-1,h=e[0],g=x.isFunction(h);if(g||!(1>=c||"string"!=typeof h||x.support.checkClone)&&Nt.test(h))return this.each(function(r){var i=p.eq(r);g&&(e[0]=h.call(this,r,i.html())),i.domManip(e,t,n)});if(c&&(l=x.buildFragment(e,this[0].ownerDocument,!1,!n&&this),r=l.firstChild,1===l.childNodes.length&&(l=r),r)){for(a=x.map(Ft(l,"script"),Ht),o=a.length;c>u;u++)i=l,u!==f&&(i=x.clone(i,!0,!0),o&&x.merge(a,Ft(i,"script"))),t.call(this[u],i,u);if(o)for(s=a[a.length-1].ownerDocument,x.map(a,qt),u=0;o>u;u++)i=a[u],kt.test(i.type||"")&&!x._data(i,"globalEval")&&x.contains(s,i)&&(i.src?x._evalUrl(i.src):x.globalEval((i.text||i.textContent||i.innerHTML||"").replace(St,"")));l=r=null}return this}});function Lt(e,t){return x.nodeName(e,"table")&&x.nodeName(1===t.nodeType?t:t.firstChild,"tr")?e.getElementsByTagName("tbody")[0]||e.appendChild(e.ownerDocument.createElement("tbody")):e}function Ht(e){return e.type=(null!==x.find.attr(e,"type"))+"/"+e.type,e}function qt(e){var t=Et.exec(e.type);return t?e.type=t[1]:e.removeAttribute("type"),e}function _t(e,t){var n,r=0;for(;null!=(n=e[r]);r++)x._data(n,"globalEval",!t||x._data(t[r],"globalEval"))}function Mt(e,t){if(1===t.nodeType&&x.hasData(e)){var n,r,i,o=x._data(e),a=x._data(t,o),s=o.events;if(s){delete a.handle,a.events={};for(n in s)for(r=0,i=s[n].length;i>r;r++)x.event.add(t,n,s[n][r])}a.data&&(a.data=x.extend({},a.data))}}function Ot(e,t){var n,r,i;if(1===t.nodeType){if(n=t.nodeName.toLowerCase(),!x.support.noCloneEvent&&t[x.expando]){i=x._data(t);for(r in i.events)x.removeEvent(t,r,i.handle);t.removeAttribute(x.expando)}"script"===n&&t.text!==e.text?(Ht(t).text=e.text,qt(t)):"object"===n?(t.parentNode&&(t.outerHTML=e.outerHTML),x.support.html5Clone&&e.innerHTML&&!x.trim(t.innerHTML)&&(t.innerHTML=e.innerHTML)):"input"===n&&Ct.test(e.type)?(t.defaultChecked=t.checked=e.checked,t.value!==e.value&&(t.value=e.value)):"option"===n?t.defaultSelected=t.selected=e.defaultSelected:("input"===n||"textarea"===n)&&(t.defaultValue=e.defaultValue)}}x.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,t){x.fn[e]=function(e){var n,r=0,i=[],o=x(e),a=o.length-1;for(;a>=r;r++)n=r===a?this:this.clone(!0),x(o[r])[t](n),h.apply(i,n.get());return this.pushStack(i)}});function Ft(e,n){var r,o,a=0,s=typeof e.getElementsByTagName!==i?e.getElementsByTagName(n||"*"):typeof e.querySelectorAll!==i?e.querySelectorAll(n||"*"):t;if(!s)for(s=[],r=e.childNodes||e;null!=(o=r[a]);a++)!n||x.nodeName(o,n)?s.push(o):x.merge(s,Ft(o,n));return n===t||n&&x.nodeName(e,n)?x.merge([e],s):s}function Bt(e){Ct.test(e.type)&&(e.defaultChecked=e.checked)}x.extend({clone:function(e,t,n){var r,i,o,a,s,l=x.contains(e.ownerDocument,e);if(x.support.html5Clone||x.isXMLDoc(e)||!mt.test("<"+e.nodeName+">")?o=e.cloneNode(!0):(Dt.innerHTML=e.outerHTML,Dt.removeChild(o=Dt.firstChild)),!(x.support.noCloneEvent&&x.support.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||x.isXMLDoc(e)))for(r=Ft(o),s=Ft(e),a=0;null!=(i=s[a]);++a)r[a]&&Ot(i,r[a]);if(t)if(n)for(s=s||Ft(e),r=r||Ft(o),a=0;null!=(i=s[a]);a++)Mt(i,r[a]);else Mt(e,o);return r=Ft(o,"script"),r.length>0&&_t(r,!l&&Ft(e,"script")),r=s=i=null,o},buildFragment:function(e,t,n,r){var i,o,a,s,l,u,c,p=e.length,f=dt(t),d=[],h=0;for(;p>h;h++)if(o=e[h],o||0===o)if("object"===x.type(o))x.merge(d,o.nodeType?[o]:o);else if(wt.test(o)){s=s||f.appendChild(t.createElement("div")),l=(bt.exec(o)||["",""])[1].toLowerCase(),c=At[l]||At._default,s.innerHTML=c[1]+o.replace(vt,"<$1>")+c[2],i=c[0];while(i--)s=s.lastChild;if(!x.support.leadingWhitespace&&yt.test(o)&&d.push(t.createTextNode(yt.exec(o)[0])),!x.support.tbody){o="table"!==l||xt.test(o)?""!==c[1]||xt.test(o)?0:s:s.firstChild,i=o&&o.childNodes.length;while(i--)x.nodeName(u=o.childNodes[i],"tbody")&&!u.childNodes.length&&o.removeChild(u)}x.merge(d,s.childNodes),s.textContent="";while(s.firstChild)s.removeChild(s.firstChild);s=f.lastChild}else d.push(t.createTextNode(o));s&&f.removeChild(s),x.support.appendChecked||x.grep(Ft(d,"input"),Bt),h=0;while(o=d[h++])if((!r||-1===x.inArray(o,r))&&(a=x.contains(o.ownerDocument,o),s=Ft(f.appendChild(o),"script"),a&&_t(s),n)){i=0;while(o=s[i++])kt.test(o.type||"")&&n.push(o)}return s=null,f},cleanData:function(e,t){var n,r,o,a,s=0,l=x.expando,u=x.cache,c=x.support.deleteExpando,f=x.event.special;for(;null!=(n=e[s]);s++)if((t||x.acceptData(n))&&(o=n[l],a=o&&u[o])){if(a.events)for(r in a.events)f[r]?x.event.remove(n,r):x.removeEvent(n,r,a.handle);u[o]&&(delete u[o],c?delete n[l]:typeof n.removeAttribute!==i?n.removeAttribute(l):n[l]=null,p.push(o))}},_evalUrl:function(e){return x.ajax({url:e,type:"GET",dataType:"script",async:!1,global:!1,"throws":!0}) -}}),x.fn.extend({wrapAll:function(e){if(x.isFunction(e))return this.each(function(t){x(this).wrapAll(e.call(this,t))});if(this[0]){var t=x(e,this[0].ownerDocument).eq(0).clone(!0);this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstChild&&1===e.firstChild.nodeType)e=e.firstChild;return e}).append(this)}return this},wrapInner:function(e){return x.isFunction(e)?this.each(function(t){x(this).wrapInner(e.call(this,t))}):this.each(function(){var t=x(this),n=t.contents();n.length?n.wrapAll(e):t.append(e)})},wrap:function(e){var t=x.isFunction(e);return this.each(function(n){x(this).wrapAll(t?e.call(this,n):e)})},unwrap:function(){return this.parent().each(function(){x.nodeName(this,"body")||x(this).replaceWith(this.childNodes)}).end()}});var Pt,Rt,Wt,$t=/alpha\([^)]*\)/i,It=/opacity\s*=\s*([^)]*)/,zt=/^(top|right|bottom|left)$/,Xt=/^(none|table(?!-c[ea]).+)/,Ut=/^margin/,Vt=RegExp("^("+w+")(.*)$","i"),Yt=RegExp("^("+w+")(?!px)[a-z%]+$","i"),Jt=RegExp("^([+-])=("+w+")","i"),Gt={BODY:"block"},Qt={position:"absolute",visibility:"hidden",display:"block"},Kt={letterSpacing:0,fontWeight:400},Zt=["Top","Right","Bottom","Left"],en=["Webkit","O","Moz","ms"];function tn(e,t){if(t in e)return t;var n=t.charAt(0).toUpperCase()+t.slice(1),r=t,i=en.length;while(i--)if(t=en[i]+n,t in e)return t;return r}function nn(e,t){return e=t||e,"none"===x.css(e,"display")||!x.contains(e.ownerDocument,e)}function rn(e,t){var n,r,i,o=[],a=0,s=e.length;for(;s>a;a++)r=e[a],r.style&&(o[a]=x._data(r,"olddisplay"),n=r.style.display,t?(o[a]||"none"!==n||(r.style.display=""),""===r.style.display&&nn(r)&&(o[a]=x._data(r,"olddisplay",ln(r.nodeName)))):o[a]||(i=nn(r),(n&&"none"!==n||!i)&&x._data(r,"olddisplay",i?n:x.css(r,"display"))));for(a=0;s>a;a++)r=e[a],r.style&&(t&&"none"!==r.style.display&&""!==r.style.display||(r.style.display=t?o[a]||"":"none"));return e}x.fn.extend({css:function(e,n){return x.access(this,function(e,n,r){var i,o,a={},s=0;if(x.isArray(n)){for(o=Rt(e),i=n.length;i>s;s++)a[n[s]]=x.css(e,n[s],!1,o);return a}return r!==t?x.style(e,n,r):x.css(e,n)},e,n,arguments.length>1)},show:function(){return rn(this,!0)},hide:function(){return rn(this)},toggle:function(e){var t="boolean"==typeof e;return this.each(function(){(t?e:nn(this))?x(this).show():x(this).hide()})}}),x.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Wt(e,"opacity");return""===n?"1":n}}}},cssNumber:{columnCount:!0,fillOpacity:!0,fontWeight:!0,lineHeight:!0,opacity:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":x.support.cssFloat?"cssFloat":"styleFloat"},style:function(e,n,r,i){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var o,a,s,l=x.camelCase(n),u=e.style;if(n=x.cssProps[l]||(x.cssProps[l]=tn(u,l)),s=x.cssHooks[n]||x.cssHooks[l],r===t)return s&&"get"in s&&(o=s.get(e,!1,i))!==t?o:u[n];if(a=typeof r,"string"===a&&(o=Jt.exec(r))&&(r=(o[1]+1)*o[2]+parseFloat(x.css(e,n)),a="number"),!(null==r||"number"===a&&isNaN(r)||("number"!==a||x.cssNumber[l]||(r+="px"),x.support.clearCloneStyle||""!==r||0!==n.indexOf("background")||(u[n]="inherit"),s&&"set"in s&&(r=s.set(e,r,i))===t)))try{u[n]=r}catch(c){}}},css:function(e,n,r,i){var o,a,s,l=x.camelCase(n);return n=x.cssProps[l]||(x.cssProps[l]=tn(e.style,l)),s=x.cssHooks[n]||x.cssHooks[l],s&&"get"in s&&(a=s.get(e,!0,r)),a===t&&(a=Wt(e,n,i)),"normal"===a&&n in Kt&&(a=Kt[n]),""===r||r?(o=parseFloat(a),r===!0||x.isNumeric(o)?o||0:a):a}}),e.getComputedStyle?(Rt=function(t){return e.getComputedStyle(t,null)},Wt=function(e,n,r){var i,o,a,s=r||Rt(e),l=s?s.getPropertyValue(n)||s[n]:t,u=e.style;return s&&(""!==l||x.contains(e.ownerDocument,e)||(l=x.style(e,n)),Yt.test(l)&&Ut.test(n)&&(i=u.width,o=u.minWidth,a=u.maxWidth,u.minWidth=u.maxWidth=u.width=l,l=s.width,u.width=i,u.minWidth=o,u.maxWidth=a)),l}):a.documentElement.currentStyle&&(Rt=function(e){return e.currentStyle},Wt=function(e,n,r){var i,o,a,s=r||Rt(e),l=s?s[n]:t,u=e.style;return null==l&&u&&u[n]&&(l=u[n]),Yt.test(l)&&!zt.test(n)&&(i=u.left,o=e.runtimeStyle,a=o&&o.left,a&&(o.left=e.currentStyle.left),u.left="fontSize"===n?"1em":l,l=u.pixelLeft+"px",u.left=i,a&&(o.left=a)),""===l?"auto":l});function on(e,t,n){var r=Vt.exec(t);return r?Math.max(0,r[1]-(n||0))+(r[2]||"px"):t}function an(e,t,n,r,i){var o=n===(r?"border":"content")?4:"width"===t?1:0,a=0;for(;4>o;o+=2)"margin"===n&&(a+=x.css(e,n+Zt[o],!0,i)),r?("content"===n&&(a-=x.css(e,"padding"+Zt[o],!0,i)),"margin"!==n&&(a-=x.css(e,"border"+Zt[o]+"Width",!0,i))):(a+=x.css(e,"padding"+Zt[o],!0,i),"padding"!==n&&(a+=x.css(e,"border"+Zt[o]+"Width",!0,i)));return a}function sn(e,t,n){var r=!0,i="width"===t?e.offsetWidth:e.offsetHeight,o=Rt(e),a=x.support.boxSizing&&"border-box"===x.css(e,"boxSizing",!1,o);if(0>=i||null==i){if(i=Wt(e,t,o),(0>i||null==i)&&(i=e.style[t]),Yt.test(i))return i;r=a&&(x.support.boxSizingReliable||i===e.style[t]),i=parseFloat(i)||0}return i+an(e,t,n||(a?"border":"content"),r,o)+"px"}function ln(e){var t=a,n=Gt[e];return n||(n=un(e,t),"none"!==n&&n||(Pt=(Pt||x("