Apply ruff C4 autofixes for perf improvement (#1352)

fsspec/asyn.py

@@ -833,7 +833,7 @@ async def _du(self, path, total=True, maxdepth=None, **kwargs):
 
     async def _find(self, path, maxdepth=None, withdirs=False, **kwargs):
         path = self._strip_protocol(path)
-        out = dict()
+        out = {}
         detail = kwargs.pop("detail", False)
 
         # Add the root directory if withdirs is requested
@@ -891,7 +891,7 @@ async def _expand_path(self, path, recursive=False, maxdepth=None):
                     out.add(p)
         if not out:
             raise FileNotFoundError(path)
-        return list(sorted(out))
+        return sorted(out)
 
     async def _mkdir(self, path, create_parents=True, **kwargs):
         pass  # not necessary to implement, may not have directories

fsspec/caching.py

@@ -456,7 +456,7 @@ def __init__(self, blocksize, fetcher, size, data={}, strict=True, **_):
 
         # simple consolidation of contiguous blocks
         if data:
-            old_offsets = sorted(list(data.keys()))
+            old_offsets = sorted(data.keys())
             offsets = [old_offsets[0]]
             blocks = [data.pop(old_offsets[0])]
             for start, stop in old_offsets[1:]:

fsspec/core.py

@@ -625,7 +625,7 @@ def get_fs_token_paths(
         pchains = [
             _un_chain(stringify_path(u), storage_options or {})[0] for u in urlpath
         ]
-        if len(set(pc[1] for pc in pchains)) > 1:
+        if len({pc[1] for pc in pchains}) > 1:
             raise ValueError("Protocol mismatch getting fs from %s", urlpath)
         paths = [pc[0] for pc in pchains]
     else:

fsspec/gui.py

@@ -257,7 +257,7 @@ def _setup(self):
             width_policy="max",
         )
         self.protocol = pn.widgets.Select(
-            options=list(sorted(known_implementations)),
+            options=sorted(known_implementations),
             value=self.init_protocol,
             name="protocol",
             align="center",

fsspec/implementations/http.py

@@ -192,7 +192,7 @@ async def _ls_real(self, url, detail=True, **kwargs):
                 for u in out
             ]
         else:
-            return list(sorted(out))
+            return sorted(out)
 
     async def _ls(self, url, detail=True, **kwargs):
         if self.use_listings_cache and url in self.dircache:

fsspec/implementations/reference.py

@@ -150,7 +150,7 @@ def listdir(self, basename=True):
         """List top-level directories"""
         if self.dirs is None:
             dirs = [p.split("/", 1)[0] for p in self.zmetadata]
-            self.dirs = set(sorted(p for p in dirs if p and not p.startswith(".")))
+            self.dirs = {p for p in dirs if p and not p.startswith(".")}
         listing = self.dirs
         if basename:
             listing = [os.path.basename(path) for path in listing]
@@ -381,17 +381,17 @@ def write(self, field, record, base_url=None, storage_options=None):
                 raws[j] = kerchunk.df._proc_raw(data)
         # TODO: only save needed columns
         df = pd.DataFrame(
-            dict(
-                path=paths,
-                offset=offsets,
-                size=sizes,
-                raw=raws,
-            ),
+            {
+                "path": paths,
+                "offset": offsets,
+                "size": sizes,
+                "raw": raws,
+            },
             copy=False,
         )
         if df.path.count() / (df.path.nunique() or 1) > self.cat_thresh:
             df["path"] = df["path"].astype("category")
-        object_encoding = dict(raw="bytes", path="utf8")
+        object_encoding = {"raw": "bytes", "path": "utf8"}
         has_nulls = ["path", "raw"]
 
         self.fs.mkdirs(f"{base_url or self.out_root}/{field}", exist_ok=True)

fsspec/implementations/tests/test_archive.py

@@ -336,7 +336,7 @@ def test_info(self, scenario: ArchiveTestScenario):
         # https://github.com/Suor/funcy/blob/1.15/funcy/colls.py#L243-L245
         def project(mapping, keys):
             """Leaves only given keys in mapping."""
-            return dict((k, mapping[k]) for k in keys if k in mapping)
+            return {k: mapping[k] for k in keys if k in mapping}
 
         with scenario.provider(archive_data) as archive:
             fs = fsspec.filesystem(scenario.protocol, fo=archive)

fsspec/implementations/tests/test_cached.py

@@ -180,12 +180,17 @@ def test_blockcache_workflow(ftp_writable, tmp_path):
     with fs.open("/out", "wb") as f:
         f.write(b"test\n" * 4096)
 
-    fs_kwargs = dict(
-        skip_instance_cache=True,
-        cache_storage=str(tmp_path),
-        target_protocol="ftp",
-        target_options={"host": host, "port": port, "username": user, "password": pw},
-    )
+    fs_kwargs = {
+        "skip_instance_cache": True,
+        "cache_storage": str(tmp_path),
+        "target_protocol": "ftp",
+        "target_options": {
+            "host": host,
+            "port": port,
+            "username": user,
+            "password": pw,
+        },
+    }
 
     # Open the blockcache and read a little bit of the data
     fs = fsspec.filesystem("blockcache", **fs_kwargs)
@@ -865,7 +870,7 @@ def test_with_compression(impl, compression):
         "%s::%s" % (impl, fn),
         "rb",
         compression=compression,
-        **{impl: dict(same_names=True, cache_storage=cachedir)},
+        **{impl: {"same_names": True, "cache_storage": cachedir}},
     ) as f:
         # stores original compressed file, uncompress on read
         assert f.read() == data
@@ -878,7 +883,11 @@ def test_with_compression(impl, compression):
         "%s::%s" % (impl, fn),
         "rb",
         **{
-            impl: dict(same_names=True, compression=compression, cache_storage=cachedir)
+            impl: {
+                "same_names": True,
+                "compression": compression,
+                "cache_storage": cachedir,
+            }
         },
     ) as f:
         # stores uncompressed data

fsspec/implementations/tests/test_local.py

@@ -145,11 +145,11 @@ def test_urlpath_expand_write():
     """Make sure * is expanded in file paths when writing."""
     _, _, paths = get_fs_token_paths("prefix-*.csv", mode="wb", num=2)
     assert all(
-        [p.endswith(pa) for p, pa in zip(paths, ["/prefix-0.csv", "/prefix-1.csv"])]
+        p.endswith(pa) for p, pa in zip(paths, ["/prefix-0.csv", "/prefix-1.csv"])
     )
     _, _, paths = get_fs_token_paths(["prefix-*.csv"], mode="wb", num=2)
     assert all(
-        [p.endswith(pa) for p, pa in zip(paths, ["/prefix-0.csv", "/prefix-1.csv"])]
+        p.endswith(pa) for p, pa in zip(paths, ["/prefix-0.csv", "/prefix-1.csv"])
     )
     # we can read with multiple masks, but not write
     with pytest.raises(ValueError):

fsspec/implementations/tests/test_sftp.py

@@ -51,7 +51,12 @@ def ssh():
         subprocess.call(["docker", "exec", cid] + shlex.split(cmd))
     try:
         time.sleep(1)
-        yield dict(host="localhost", port=9200, username="root", password="pass")
+        yield {
+            "host": "localhost",
+            "port": 9200,
+            "username": "root",
+            "password": "pass",
+        }
     finally:
         stop_docker(name)
 

fsspec/implementations/tests/test_smb.py

@@ -48,12 +48,12 @@ def smb_params(request):
     logger.debug("Container: %s", cid)
     try:
         time.sleep(1)
-        yield dict(
-            host="localhost",
-            port=request.param,
-            username="testuser",
-            password="testpass",
-        )
+        yield {
+            "host": "localhost",
+            "port": request.param,
+            "username": "testuser",
+            "password": "testpass",
+        }
     finally:
         import smbclient  # pylint: disable=import-outside-toplevel
 

fsspec/implementations/tests/test_zip.py

@@ -26,7 +26,7 @@ def test_fsspec_get_mapper():
         mapping = fsspec.get_mapper(f"zip::{z}")
 
         assert isinstance(mapping, collections.abc.Mapping)
-        keys = sorted(list(mapping.keys()))
+        keys = sorted(mapping.keys())
         assert keys == ["a", "b", "deeply/nested/path"]
 
         # mapping.getitems() will call FSMap.fs.cat()

fsspec/spec.py

@@ -485,7 +485,7 @@ def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs):
         """
         # TODO: allow equivalent of -name parameter
         path = self._strip_protocol(path)
-        out = dict()
+        out = {}
 
         # Add the root directory if withdirs is requested
         # This is needed for posix glob compliance
@@ -1182,7 +1182,7 @@ def expand_path(self, path, recursive=False, maxdepth=None, **kwargs):
                     out.add(p)
         if not out:
             raise FileNotFoundError(path)
-        return list(sorted(out))
+        return sorted(out)
 
     def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs):
         """Move file(s) from one location to another"""

fsspec/tests/test_caches.py

@@ -101,7 +101,7 @@ def test_cache_basic(Cache_imp, blocksize, size_requests):
 def test_known(sort, strict):
     parts = {(10, 20): b"1" * 10, (20, 30): b"2" * 10, (0, 10): b"0" * 10}
     if sort:
-        parts = {k: v for k, v in sorted(parts.items())}
+        parts = dict(sorted(parts.items()))
     c = caches["parts"](None, None, 100, parts, strict=strict)
     assert (0, 30) in c.data  # got consolidated
     assert c._fetch(5, 15) == b"0" * 5 + b"1" * 5

fsspec/tests/test_callbacks.py

@@ -7,11 +7,11 @@ def test_callbacks():
     empty_callback = Callback()
     assert empty_callback.call("something", somearg=None) is None
 
-    hooks = dict(something=lambda *_, arg=None: arg + 2)
+    hooks = {"something": lambda *_, arg=None: arg + 2}
     simple_callback = Callback(hooks=hooks)
     assert simple_callback.call("something", arg=2) == 4
 
-    hooks = dict(something=lambda *_, arg1=None, arg2=None: arg1 + arg2)
+    hooks = {"something": lambda *_, arg1=None, arg2=None: arg1 + arg2}
     multi_arg_callback = Callback(hooks=hooks)
     assert multi_arg_callback.call("something", arg1=2, arg2=2) == 4
 
@@ -21,7 +21,7 @@ def test_callbacks_as_callback():
     assert empty_callback.call("something", arg="somearg") is None
     assert Callback.as_callback(None) is Callback.as_callback(None)
 
-    hooks = dict(something=lambda *_, arg=None: arg + 2)
+    hooks = {"something": lambda *_, arg=None: arg + 2}
     real_callback = Callback.as_callback(Callback(hooks=hooks))
     assert real_callback.call("something", arg=2) == 4
 

fsspec/tests/test_downstream.py

@@ -10,7 +10,7 @@
     test_bucket_name,
 )
 
-so = dict(anon=False, client_kwargs={"endpoint_url": endpoint_uri})
+so = {"anon": False, "client_kwargs": {"endpoint_url": endpoint_uri}}
 
 
 def test_pandas(s3):

fsspec/tests/test_fuse.py

@@ -127,7 +127,7 @@ def test_chmod(mount_local):
 
     assert cp.stderr == b""
     assert cp.stdout == b""
-    assert set(os.listdir(source_dir)) == set(["text", "new"])
+    assert set(os.listdir(source_dir)) == {"text", "new"}
     assert open(mount_dir / "new").read() == "test"
 
 

fsspec/tests/test_spec.py

@@ -765,11 +765,11 @@ def test_cache_options():
 
     # TODO: dummy buffered file
     f = AbstractBufferedFile(
-        fs, "misc/foo.txt", cache_type="bytes", cache_options=dict(trim=False)
+        fs, "misc/foo.txt", cache_type="bytes", cache_options={"trim": False}
     )
     assert f.cache.trim is False
 
-    f = fs.open("misc/foo.txt", cache_type="bytes", cache_options=dict(trim=False))
+    f = fs.open("misc/foo.txt", cache_type="bytes", cache_options={"trim": False})
     assert f.cache.trim is False
 
 
@@ -831,7 +831,7 @@ def test_pickle_multiple():
 
     result = pickle.loads(y)
     assert result.storage_args == (2,)
-    assert result.storage_options == dict(bar=1)
+    assert result.storage_options == {"bar": 1}
 
 
 def test_json():