diff --git a/.flake8 b/.flake8
index dae85fef0d..7a617508f2 100644
--- a/.flake8
+++ b/.flake8
@@ -6,7 +6,7 @@ ignore = E203,E402,E741,W503
 
 # Note: exclude is not honnored when flake8 is executed from pre-commit.
 # pre-commit has a separate config
-exclude = build,docs/src,third_party
+exclude = build,docs/src
 
 per-file-ignores =
   examples/tutorials/*.py: E501
diff --git a/.github/scripts/unittest-linux/run_test.sh b/.github/scripts/unittest-linux/run_test.sh
index 6cc935b444..0485a3617f 100755
--- a/.github/scripts/unittest-linux/run_test.sh
+++ b/.github/scripts/unittest-linux/run_test.sh
@@ -9,7 +9,6 @@ conda activate ci
 python -m torch.utils.collect_env
 env | grep TORCHAUDIO || true
 
-export PATH="${PWD}/third_party/install/bin/:${PATH}"
 
 declare -a args=(
     '--continue-on-collection-errors'
diff --git a/.github/workflows/unittest-linux-cpu.yml b/.github/workflows/unittest-linux-cpu.yml
index 99f7875bde..f34fb4c255 100644
--- a/.github/workflows/unittest-linux-cpu.yml
+++ b/.github/workflows/unittest-linux-cpu.yml
@@ -43,7 +43,6 @@ jobs:
           export UPLOAD_CHANNEL=nightly
         fi
 
-        export PATH="${PWD}/third_party/install/bin/:${PATH}"
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS=true
diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml
index 3d785db097..91b79f5b47 100644
--- a/.github/workflows/unittest-linux-gpu.yml
+++ b/.github/workflows/unittest-linux-gpu.yml
@@ -81,7 +81,6 @@ jobs:
 
         echo "::endgroup::"
         echo "::group::Run tests"
-        export PATH="${PWD}/third_party/install/bin/:${PATH}"
 
         declare -a args=(
             '-v'
diff --git a/.gitignore b/.gitignore
index 8dadcd51f6..f778f1ff18 100644
--- a/.gitignore
+++ b/.gitignore
@@ -127,7 +127,3 @@ examples/interactive_asr/data/*.txt
 examples/interactive_asr/data/*.model
 examples/interactive_asr/data/*.pt
 examples/tutorials/_assets
-
-# third parties
-third_party/install/
-third_party/archives/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a96eacb341..e0bd36e4e9 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -30,7 +30,7 @@ repos:
     hooks:
       - id: flake8
         args: ['src', 'test', 'tools', 'docs/source/conf.py', 'examples']
-        exclude: 'build|docs/src|third_party'
+        exclude: 'build|docs/src'
         additional_dependencies:
           - flake8-breakpoint == 1.1.0
           - flake8-bugbear == 22.6.22
@@ -43,4 +43,4 @@ repos:
     rev: 6.3.0
     hooks:
       - id: pydocstyle
-        exclude: 'build|test|examples|third_party|docs|tools'
+        exclude: 'build|test|examples|docs|tools'
diff --git a/third_party/ffmpeg/multi/CMakeLists.txt b/third_party/ffmpeg/multi/CMakeLists.txt
deleted file mode 100644
index 88f51adb94..0000000000
--- a/third_party/ffmpeg/multi/CMakeLists.txt
+++ /dev/null
@@ -1,198 +0,0 @@
-################################################################################
-# This file defines the following FFmpeg libraries using pre-built binaries.
-
-add_library(ffmpeg4 INTERFACE)
-add_library(ffmpeg5 INTERFACE)
-add_library(ffmpeg6 INTERFACE)
-
-################################################################################
-
-include(FetchContent)
-
-set(base_url https://pytorch.s3.amazonaws.com/torchaudio/ffmpeg)
-
-if (APPLE)
-  if ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "arm64")
-    FetchContent_Declare(
-      f4
-      URL ${base_url}/2023-08-14/macos_arm64/4.4.4.tar.gz
-      URL_HASH SHA256=9a593eb241eb8b23bc557856ee6db5d9aecd2d8895c614a949f3a1ad9799c1a1
-      )
-    FetchContent_Declare(
-      f5
-      URL ${base_url}/2023-07-06/macos_arm64/5.0.3.tar.gz
-      URL_HASH SHA256=316fe8378afadcf63089acf3ad53a626fd3c26cc558b96ce1dc94d2a78f4deb4
-      )
-    FetchContent_Declare(
-      f6
-      URL ${base_url}/2023-07-06/macos_arm64/6.0.tar.gz
-      URL_HASH SHA256=5d1da9626f8cb817d6c558a2c61085a3d39a8d9f725a6f69f4658bea8efa9389
-      )
-  elseif ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64")
-    FetchContent_Declare(
-      f4
-      URL ${base_url}/2023-08-14/macos_x86_64/4.4.4.tar.gz
-      URL_HASH SHA256=0935e359c0864969987d908397f9208d6dc4dc0ef8bfe2ec730bb2c44eae89fc
-      )
-    FetchContent_Declare(
-      f5
-      URL ${base_url}/2023-07-06/macos_x86_64/5.0.3.tar.gz
-      URL_HASH SHA256=d0b49575d3b174cfcca53b3049641855e48028cf22dd32f3334bbec4ca94f43e
-      )
-    FetchContent_Declare(
-      f6
-      URL ${base_url}/2023-07-06/macos_x86_64/6.0.tar.gz
-      URL_HASH SHA256=eabc01eb7d9e714e484d5e1b27bf7d921e87c1f3c00334abd1729e158d6db862
-      )
-  else ()
-    message(
-      FATAL_ERROR
-      "${CMAKE_SYSTEM_PROCESSOR} is not supported for FFmpeg multi-version integration. "
-      "If you have FFmpeg libraries installed in the system,"
-      " setting FFMPEG_ROOT environment variable to the root directory of FFmpeg installation"
-      " (the directory where `include` and `lib` sub directories with corresponding headers"
-      " and library files are present) will invoke the FFmpeg single-version integration. "
-      "If you do not need the FFmpeg integration, setting USE_FFMPEG=0 will bypass the issue.")
-  endif()
-elseif (UNIX)
-  if ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64")
-    FetchContent_Declare(
-      f4
-      URL ${base_url}/2023-08-14/linux_aarch64/4.4.4.tar.gz
-      URL_HASH SHA256=6f00437d13a3b3812ebe81c6e6f3a84a58f260d946a1995df87ba09aae234504
-      )
-    FetchContent_Declare(
-      f5
-      URL ${base_url}/2023-07-06/linux_aarch64/5.0.3.tar.gz
-      URL_HASH SHA256=65c663206982ee3f0ff88436d8869d191b46061e01e753518c77ecc13ea0236d
-      )
-    FetchContent_Declare(
-      f6
-      URL ${base_url}/2023-07-06/linux_aarch64/6.0.tar.gz
-      URL_HASH SHA256=ec762fd41ea7b8d9ad4f810f53fd78a565f2bc6f680afe56d555c80f3d35adef
-      )
-  elseif ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64")
-    FetchContent_Declare(
-      f4
-      URL ${base_url}/2023-08-14/linux_x86_64/4.4.4.tar.gz
-      URL_HASH SHA256=9b87eeba9b6975e25f28ba12163bd713228ed84f4c2b3721bc5ebe92055edb51
-      )
-    FetchContent_Declare(
-      f5
-      URL ${base_url}/2023-07-06/linux_x86_64/5.0.3.tar.gz
-      URL_HASH SHA256=de3c75c99b9ce33de7efdc144566804ae5880457ce71e185b3f592dc452edce7
-      )
-    FetchContent_Declare(
-      f6
-      URL ${base_url}/2023-07-06/linux_x86_64/6.0.tar.gz
-      URL_HASH SHA256=04d3916404bab5efadd29f68361b7d13ea71e6242c6473edcb747a41a9fb97a6
-      )
-  endif()
-elseif(MSVC)
-  FetchContent_Declare(
-    f4
-    URL ${base_url}/2023-08-14/windows/4.4.4.tar.gz
-    URL_HASH SHA256=9f9a65cf03a3e164edca601ba18180a504e44e03fae48ce706ca3120b55a4db5
-    )
-  FetchContent_Declare(
-    f5
-    URL ${base_url}/2023-07-06/windows/5.0.3.tar.gz
-    URL_HASH SHA256=e2daa10799909e366cb1b4b91a217d35f6749290dcfeea40ecae3d5b05a46cb3
-    )
-  FetchContent_Declare(
-    f6
-    URL ${base_url}/2023-07-06/windows/6.0.tar.gz
-    URL_HASH SHA256=098347eca8cddb5aaa61e9ecc1a00548c645fc59b4f7346b3d91414aa00a9cf6
-    )
-endif()
-
-FetchContent_MakeAvailable(f4 f5 f6)
-target_include_directories(ffmpeg4 INTERFACE ${f4_SOURCE_DIR}/include)
-target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include)
-target_include_directories(ffmpeg6 INTERFACE ${f6_SOURCE_DIR}/include)
-
-if(APPLE)
-  target_link_libraries(
-    ffmpeg4
-    INTERFACE
-    ${f4_SOURCE_DIR}/lib/libavutil.56.dylib
-    ${f4_SOURCE_DIR}/lib/libavcodec.58.dylib
-    ${f4_SOURCE_DIR}/lib/libavformat.58.dylib
-    ${f4_SOURCE_DIR}/lib/libavdevice.58.dylib
-    ${f4_SOURCE_DIR}/lib/libavfilter.7.dylib
-    )
-  target_link_libraries(
-    ffmpeg5
-    INTERFACE
-    ${f5_SOURCE_DIR}/lib/libavutil.57.dylib
-    ${f5_SOURCE_DIR}/lib/libavcodec.59.dylib
-    ${f5_SOURCE_DIR}/lib/libavformat.59.dylib
-    ${f5_SOURCE_DIR}/lib/libavdevice.59.dylib
-    ${f5_SOURCE_DIR}/lib/libavfilter.8.dylib
-    )
-  target_link_libraries(
-    ffmpeg6
-    INTERFACE
-    ${f6_SOURCE_DIR}/lib/libavutil.58.dylib
-    ${f6_SOURCE_DIR}/lib/libavcodec.60.dylib
-    ${f6_SOURCE_DIR}/lib/libavformat.60.dylib
-    ${f6_SOURCE_DIR}/lib/libavdevice.60.dylib
-    ${f6_SOURCE_DIR}/lib/libavfilter.9.dylib
-    )
-elseif (UNIX)
-  target_link_libraries(
-    ffmpeg4
-    INTERFACE
-    ${f4_SOURCE_DIR}/lib/libavutil.so.56
-    ${f4_SOURCE_DIR}/lib/libavcodec.so.58
-    ${f4_SOURCE_DIR}/lib/libavformat.so.58
-    ${f4_SOURCE_DIR}/lib/libavdevice.so.58
-    ${f4_SOURCE_DIR}/lib/libavfilter.so.7
-    )
-  target_link_libraries(
-    ffmpeg5
-    INTERFACE
-    ${f5_SOURCE_DIR}/lib/libavutil.so.57
-    ${f5_SOURCE_DIR}/lib/libavcodec.so.59
-    ${f5_SOURCE_DIR}/lib/libavformat.so.59
-    ${f5_SOURCE_DIR}/lib/libavdevice.so.59
-    ${f5_SOURCE_DIR}/lib/libavfilter.so.8
-    )
-  target_link_libraries(
-    ffmpeg6
-    INTERFACE
-    ${f6_SOURCE_DIR}/lib/libavutil.so.58
-    ${f6_SOURCE_DIR}/lib/libavcodec.so.60
-    ${f6_SOURCE_DIR}/lib/libavformat.so.60
-    ${f6_SOURCE_DIR}/lib/libavdevice.so.60
-    ${f6_SOURCE_DIR}/lib/libavfilter.so.9
-    )
-elseif(MSVC)
-  target_link_libraries(
-    ffmpeg4
-    INTERFACE
-    ${f4_SOURCE_DIR}/bin/avutil.lib
-    ${f4_SOURCE_DIR}/bin/avcodec.lib
-    ${f4_SOURCE_DIR}/bin/avformat.lib
-    ${f4_SOURCE_DIR}/bin/avdevice.lib
-    ${f4_SOURCE_DIR}/bin/avfilter.lib
-    )
-  target_link_libraries(
-    ffmpeg5
-    INTERFACE
-    ${f5_SOURCE_DIR}/bin/avutil.lib
-    ${f5_SOURCE_DIR}/bin/avcodec.lib
-    ${f5_SOURCE_DIR}/bin/avformat.lib
-    ${f5_SOURCE_DIR}/bin/avdevice.lib
-    ${f5_SOURCE_DIR}/bin/avfilter.lib
-    )
-  target_link_libraries(
-    ffmpeg6
-    INTERFACE
-    ${f6_SOURCE_DIR}/bin/avutil.lib
-    ${f6_SOURCE_DIR}/bin/avcodec.lib
-    ${f6_SOURCE_DIR}/bin/avformat.lib
-    ${f6_SOURCE_DIR}/bin/avdevice.lib
-    ${f6_SOURCE_DIR}/bin/avfilter.lib
-    )
-endif()
diff --git a/third_party/ffmpeg/single/CMakeLists.txt b/third_party/ffmpeg/single/CMakeLists.txt
deleted file mode 100644
index b3246c0828..0000000000
--- a/third_party/ffmpeg/single/CMakeLists.txt
+++ /dev/null
@@ -1,40 +0,0 @@
-# CMake file for searching existing FFmpeg installation and defining ffmpeg TARGET
-
-message(STATUS "Searching existing FFmpeg installation")
-message(STATUS FFMPEG_ROOT=$ENV{FFMPEG_ROOT})
-if (NOT DEFINED ENV{FFMPEG_ROOT})
-  message(FATAL_ERROR "Environment variable FFMPEG_ROOT is not set.")
-endif()
-
-set(_root $ENV{FFMPEG_ROOT})
-set(lib_dirs "${_root}/lib" "${_root}/bin")
-set(include_dir "${_root}/include")
-
-add_library(ffmpeg INTERFACE)
-target_include_directories(ffmpeg INTERFACE "${include_dir}")
-
-function (_find_ffmpeg_lib component)
-  find_path("${component}_header"
-    NAMES "lib${component}/${component}.h"
-    PATHS "${include_dir}"
-    DOC "The include directory for ${component}"
-    REQUIRED
-    NO_DEFAULT_PATH)
-  find_library("lib${component}"
-    NAMES "${component}"
-    PATHS ${lib_dirs}
-    DOC "${component} library"
-    REQUIRED
-    NO_DEFAULT_PATH)
-  message(STATUS "Found ${component}: ${lib${component}}")
-  target_link_libraries(
-    ffmpeg
-    INTERFACE
-    ${lib${component}})
-endfunction ()
-
-_find_ffmpeg_lib(avutil)
-_find_ffmpeg_lib(avcodec)
-_find_ffmpeg_lib(avformat)
-_find_ffmpeg_lib(avdevice)
-_find_ffmpeg_lib(avfilter)