Coverage for datakiln/metadata/files.py: 49%
53 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
1"""
2Data files of the tables.
3"""
4from hashlib import md5
5from pathlib import Path
6from typing import List
8from dvc.api import DVCFileSystem
11class LocalFile(dict):
12 """Local file or folder, with the keys of one that DVC tracks. Its
13 checksum is computed when it is needed, as that takes reading it."""
15 def __init__(self, path: Path, repo: 'LocalRepo'):
16 super().__init__(name=path.name, repo=repo)
17 self._path = path
19 def __missing__(self, key):
20 if key != "md5":
21 raise KeyError(key)
23 # A folder is told by the names and the content of its files
24 files = [self._path]
25 if self._path.is_dir():
26 files = sorted(self._path.rglob("*"))
28 checksum = md5() # noqa: DUO130
29 for file in files:
30 if file.is_dir():
31 continue
32 if self._path.is_dir():
33 checksum.update(str(file.relative_to(self._path)).encode())
34 with open(file, "rb") as content:
35 while chunk := content.read(4096):
36 checksum.update(chunk)
38 self[key] = checksum.hexdigest()
39 return self[key]
42class LocalRepo:
43 """Folder of data files that DVC does not track. It has the part of a DVC
44 repository, and of its file system, that the tables use, so that there is
45 nothing to fetch."""
47 def __init__(self, root_dir: Path):
48 self.root_dir = str(root_dir)
50 def listdir(self, path: str, dvc_only: bool = False) -> List[LocalFile]:
51 # pylint: disable=unused-argument
52 """Files and folders, other than the hidden ones and the ones of DVC.
54 :param path: Folder in the root.
55 :type path: str
57 :param dvc_only: Ignored, as every file is local.
58 :type dvc_only: bool
60 :rtype: list[LocalFile]
61 """
62 files = []
63 for file in sorted((Path(self.root_dir) / path).iterdir()):
64 if not file.name.startswith(".") and file.suffix != ".dvc":
65 files.append(LocalFile(file, self))
66 return files
68 def pull(self, targets):
69 # pylint: disable=unused-argument
70 """Nothing is fetched, since the files are local.
72 :return: What was fetched, which is nothing.
73 :rtype: dict
74 """
75 return {}
78class DataFiles:
79 # pylint: disable=too-few-public-methods
80 """Data files of the tables. They are the ones DVC tracks if the folder is
81 a DVC repository, and the ones in it otherwise."""
83 def __init__(self, directory: Path):
84 """
85 :param directory: Folder of the data files.
86 :type directory: Path
87 """
88 directory = Path(directory)
89 if (directory / ".dvc").is_dir():
90 self._files = DVCFileSystem(directory)
91 else:
92 self._files = LocalRepo(directory)
94 @staticmethod
95 def _named(path: str, name: str) -> bool:
96 """Whether a file is named so, without its suffixes and ignoring the
97 case."""
98 path = Path(path)
99 suffix = "".join(path.suffixes)
100 stem = path.name[:-len(suffix)] if suffix else path.name
101 return stem.lower() == name
103 def find(self, name: str) -> dict:
104 """Find the data of a table.
106 :param name: Name of the table metadata, in lower case.
107 :type name: str
109 :return: Name of the file or folder in the folder of the data files,
110 its checksum and the repository it belongs to.
111 :rtype: dict
113 :raises FileNotFoundError: If there is no file for the table.
114 """
115 for artifact in self._files.listdir(".", dvc_only=True):
116 if self._named(artifact["name"], name):
117 return artifact
119 raise FileNotFoundError(f"No files found for metadata: '{name}'")