Fix: Monorepo dependency parsing and docs fetch pipeline integration

- Add automatic detection of package.json in common monorepo patterns (backend/, frontend/, packages/*/, apps/*/,
  services/*/)
  - Fix pipeline deps.json path mismatch - deps writes to .pf/raw/deps.json but docs fetch was looking in
  .pf/deps.json
  - Add _parse_standalone_package_json() to handle subdirectory packages without workspace detection
  - Properly track workspace_package field with relative paths for all discovered package.json files

  Fixes PlantPro and similar monorepos where deps were not being detected (only finding 1 Docker dep instead of 100+
   npm deps)
This commit is contained in:
TheAuditorTool
2025-09-09 16:48:11 +07:00
parent c7a59e420b
commit c7b1f54ab8
2 changed files with 69 additions and 1 deletions

View File

@@ -56,6 +56,42 @@ def parse_dependencies(root_path: str = ".") -> List[Dict[str, Any]]:
if debug: if debug:
print(f"Debug: Security error checking package.json: {e}") print(f"Debug: Security error checking package.json: {e}")
# Check for package.json in common monorepo patterns (even without workspaces field)
# This handles cases where monorepos don't use npm/yarn/pnpm workspaces
npm_patterns = [
"*/package.json", # backend/package.json, frontend/package.json
"packages/*/package.json", # packages/core/package.json
"apps/*/package.json", # apps/web/package.json
"services/*/package.json", # services/api/package.json
]
package_files = []
for pattern in npm_patterns:
package_files.extend(root.glob(pattern))
# Process all discovered package.json files
for pkg_file in package_files:
try:
safe_pkg = sanitize_path(str(pkg_file), root_path)
if debug:
print(f"Debug: Found {safe_pkg}")
# Parse this package.json directly without workspace detection
pkg_deps = _parse_standalone_package_json(safe_pkg)
# Set the workspace_package field to reflect the actual path
try:
rel_path = safe_pkg.relative_to(Path(root_path).resolve())
workspace_path = str(rel_path).replace("\\", "/")
except ValueError:
workspace_path = str(safe_pkg)
for dep in pkg_deps:
dep["workspace_package"] = workspace_path
deps.extend(pkg_deps)
except SecurityError as e:
if debug:
print(f"Debug: Security error with {pkg_file}: {e}")
# Parse Python dependencies # Parse Python dependencies
try: try:
pyproject = sanitize_path("pyproject.toml", root_path) pyproject = sanitize_path("pyproject.toml", root_path)
@@ -132,6 +168,38 @@ def parse_dependencies(root_path: str = ".") -> List[Dict[str, Any]]:
return deps return deps
def _parse_standalone_package_json(path: Path) -> List[Dict[str, Any]]:
"""Parse dependencies from a single package.json file without workspace detection."""
deps = []
try:
with open(path, encoding="utf-8") as f:
data = json.load(f)
# Combine dependencies and devDependencies
all_deps = {}
if "dependencies" in data:
all_deps.update(data["dependencies"])
if "devDependencies" in data:
all_deps.update(data["devDependencies"])
for name, version_spec in all_deps.items():
# Clean version spec (remove ^, ~, >=, etc.)
version = _clean_version(version_spec)
deps.append({
"name": name,
"version": version,
"manager": "npm",
"files": [], # Will be populated by workset scan
"source": "package.json",
"workspace_package": "package.json" # Will be overridden by caller
})
except (json.JSONDecodeError, KeyError) as e:
# Log but don't fail - package.json might be malformed
print(f"Warning: Could not parse {path}: {e}")
return deps
def _parse_package_json(path: Path) -> List[Dict[str, Any]]: def _parse_package_json(path: Path) -> List[Dict[str, Any]]:
"""Parse dependencies from package.json, with monorepo support.""" """Parse dependencies from package.json, with monorepo support."""
deps = [] deps = []

View File

@@ -363,7 +363,7 @@ def run_full_pipeline(
("index", []), ("index", []),
("detect-frameworks", []), ("detect-frameworks", []),
("deps", ["--check-latest"]), ("deps", ["--check-latest"]),
("docs", ["fetch"]), ("docs", ["fetch", "--deps", "./.pf/raw/deps.json"]),
("docs", ["summarize"]), ("docs", ["summarize"]),
("workset", ["--all"]), ("workset", ["--all"]),
("lint", ["--workset"]), ("lint", ["--workset"]),