Remove bulk importer
This commit is contained in:
parent
54c7ab09f7
commit
a74a7da225
|
|
@ -1,422 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
Bulk-create Jira Cloud issues from a CSV file.
|
|
||||||
|
|
||||||
CSV expected columns:
|
|
||||||
Summary,Description,Work Type,Priority,Status,Component,Component
|
|
||||||
|
|
||||||
Example:
|
|
||||||
python jira_bulk_import.py \
|
|
||||||
--site https://your-domain.atlassian.net \
|
|
||||||
--email you@example.com \
|
|
||||||
--api-token YOUR_TOKEN \
|
|
||||||
--project TURN \
|
|
||||||
--csv issues.csv \
|
|
||||||
--create-missing-components
|
|
||||||
|
|
||||||
Optional:
|
|
||||||
--default-component General
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import csv
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import requests
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class CsvIssue:
|
|
||||||
summary: str
|
|
||||||
description: str
|
|
||||||
work_type: str
|
|
||||||
priority: str
|
|
||||||
status: str
|
|
||||||
components: list[str]
|
|
||||||
|
|
||||||
|
|
||||||
class JiraClient:
|
|
||||||
def __init__(self, site: str, email: str, api_token: str, timeout: int = 30) -> None:
|
|
||||||
self.site = site.rstrip("/")
|
|
||||||
self.timeout = timeout
|
|
||||||
self.session = requests.Session()
|
|
||||||
self.session.auth = (email, api_token)
|
|
||||||
self.session.headers.update(
|
|
||||||
{
|
|
||||||
"Accept": "application/json",
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
def _url(self, path: str) -> str:
|
|
||||||
return f"{self.site}{path}"
|
|
||||||
|
|
||||||
def get(self, path: str, **kwargs: Any) -> requests.Response:
|
|
||||||
resp = self.session.get(self._url(path), timeout=self.timeout, **kwargs)
|
|
||||||
self._raise_for_status(resp)
|
|
||||||
return resp
|
|
||||||
|
|
||||||
def post(self, path: str, payload: dict[str, Any]) -> requests.Response:
|
|
||||||
resp = self.session.post(
|
|
||||||
self._url(path),
|
|
||||||
timeout=self.timeout,
|
|
||||||
data=json.dumps(payload),
|
|
||||||
)
|
|
||||||
self._raise_for_status(resp)
|
|
||||||
return resp
|
|
||||||
|
|
||||||
def _raise_for_status(self, resp: requests.Response) -> None:
|
|
||||||
if resp.ok:
|
|
||||||
return
|
|
||||||
|
|
||||||
body = resp.text
|
|
||||||
try:
|
|
||||||
parsed = resp.json()
|
|
||||||
body = json.dumps(parsed, indent=2, ensure_ascii=False)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Jira API error {resp.status_code} for {resp.request.method} {resp.url}\n{body}"
|
|
||||||
)
|
|
||||||
|
|
||||||
def get_project_components(self, project_key: str) -> dict[str, dict[str, Any]]:
|
|
||||||
resp = self.get(f"/rest/api/3/project/{project_key}/components")
|
|
||||||
items = resp.json()
|
|
||||||
return {item["name"].strip().lower(): item for item in items}
|
|
||||||
|
|
||||||
def create_component(self, project_key: str, component_name: str) -> dict[str, Any]:
|
|
||||||
payload = {
|
|
||||||
"name": component_name,
|
|
||||||
"project": project_key,
|
|
||||||
}
|
|
||||||
resp = self.post("/rest/api/3/component", payload)
|
|
||||||
return resp.json()
|
|
||||||
|
|
||||||
def get_issue_types_for_project(self, project_key: str) -> dict[str, dict[str, Any]]:
|
|
||||||
resp = self.get(f"/rest/api/3/issue/createmeta/{project_key}/issuetypes")
|
|
||||||
data = resp.json()
|
|
||||||
|
|
||||||
values = data.get("issueTypes") or data.get("values") or []
|
|
||||||
result: dict[str, dict[str, Any]] = {}
|
|
||||||
for item in values:
|
|
||||||
name = item["name"].strip().lower()
|
|
||||||
result[name] = item
|
|
||||||
return result
|
|
||||||
|
|
||||||
def get_create_fields_metadata(self, project_key: str, issue_type_id: str) -> dict[str, Any]:
|
|
||||||
resp = self.get(f"/rest/api/3/issue/createmeta/{project_key}/issuetypes/{issue_type_id}")
|
|
||||||
data = resp.json()
|
|
||||||
return data.get("fields", {})
|
|
||||||
|
|
||||||
def bulk_create_issues(self, issue_updates: list[dict[str, Any]]) -> dict[str, Any]:
|
|
||||||
payload = {"issueUpdates": issue_updates}
|
|
||||||
resp = self.post("/rest/api/3/issue/bulk", payload)
|
|
||||||
return resp.json()
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
|
||||||
parser = argparse.ArgumentParser(description="Bulk import Jira issues from CSV")
|
|
||||||
parser.add_argument("--site", required=True, help="https://your-domain.atlassian.net")
|
|
||||||
parser.add_argument("--email", required=True, help="Jira account email")
|
|
||||||
parser.add_argument("--api-token", required=True, help="Jira API token")
|
|
||||||
parser.add_argument("--project", required=True, help="Jira project key, e.g. TURN")
|
|
||||||
parser.add_argument("--csv", required=True, help="Path to CSV file")
|
|
||||||
parser.add_argument(
|
|
||||||
"--create-missing-components",
|
|
||||||
action="store_true",
|
|
||||||
help="Create missing project components automatically",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--default-component",
|
|
||||||
default="General",
|
|
||||||
help="Fallback Jira component to use when a CSV row has no component",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--dry-run",
|
|
||||||
action="store_true",
|
|
||||||
help="Print the payloads without sending them to Jira",
|
|
||||||
)
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def read_csv_issues(csv_path: str) -> list[CsvIssue]:
|
|
||||||
issues: list[CsvIssue] = []
|
|
||||||
|
|
||||||
with open(csv_path, "r", encoding="utf-8-sig", newline="") as f:
|
|
||||||
reader = csv.reader(f)
|
|
||||||
rows = list(reader)
|
|
||||||
|
|
||||||
if not rows:
|
|
||||||
raise RuntimeError("CSV is empty")
|
|
||||||
|
|
||||||
header = rows[0]
|
|
||||||
expected_prefix = ["Summary", "Description", "Work Type", "Priority", "Status", "Component"]
|
|
||||||
if len(header) < 6 or header[:6] != expected_prefix:
|
|
||||||
raise RuntimeError(
|
|
||||||
"Unexpected CSV header. Expected at least:\n"
|
|
||||||
"Summary,Description,Work Type,Priority,Status,Component,Component"
|
|
||||||
)
|
|
||||||
|
|
||||||
for idx, row in enumerate(rows[1:], start=2):
|
|
||||||
if not row or not any(cell.strip() for cell in row):
|
|
||||||
continue
|
|
||||||
|
|
||||||
padded = row + [""] * max(0, 7 - len(row))
|
|
||||||
|
|
||||||
summary = padded[0].strip()
|
|
||||||
description = padded[1].strip()
|
|
||||||
work_type = padded[2].strip()
|
|
||||||
priority = padded[3].strip()
|
|
||||||
status = padded[4].strip()
|
|
||||||
component_1 = padded[5].strip()
|
|
||||||
component_2 = padded[6].strip() if len(padded) > 6 else ""
|
|
||||||
|
|
||||||
if not summary:
|
|
||||||
print(f"Skipping row {idx}: empty Summary", file=sys.stderr)
|
|
||||||
continue
|
|
||||||
|
|
||||||
components: list[str] = []
|
|
||||||
for comp in (component_1, component_2):
|
|
||||||
comp = comp.strip()
|
|
||||||
if comp and comp not in components:
|
|
||||||
components.append(comp)
|
|
||||||
|
|
||||||
issues.append(
|
|
||||||
CsvIssue(
|
|
||||||
summary=summary,
|
|
||||||
description=description,
|
|
||||||
work_type=work_type,
|
|
||||||
priority=priority,
|
|
||||||
status=status,
|
|
||||||
components=components,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return issues
|
|
||||||
|
|
||||||
|
|
||||||
def adf_paragraph(text: str) -> dict[str, Any]:
|
|
||||||
paragraphs = []
|
|
||||||
for line in text.splitlines():
|
|
||||||
if line.strip():
|
|
||||||
paragraphs.append(
|
|
||||||
{
|
|
||||||
"type": "paragraph",
|
|
||||||
"content": [{"type": "text", "text": line}],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
paragraphs.append({"type": "paragraph", "content": []})
|
|
||||||
|
|
||||||
if not paragraphs:
|
|
||||||
paragraphs = [{"type": "paragraph", "content": []}]
|
|
||||||
|
|
||||||
return {
|
|
||||||
"type": "doc",
|
|
||||||
"version": 1,
|
|
||||||
"content": paragraphs,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def chunked[T](items: list[T], size: int) -> list[list[T]]:
|
|
||||||
return [items[i:i + size] for i in range(0, len(items), size)]
|
|
||||||
|
|
||||||
|
|
||||||
def normalize(s: str) -> str:
|
|
||||||
return s.strip().lower()
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_components(
|
|
||||||
jira: JiraClient,
|
|
||||||
project_key: str,
|
|
||||||
issues: list[CsvIssue],
|
|
||||||
create_missing: bool,
|
|
||||||
default_component: str,
|
|
||||||
) -> dict[str, dict[str, Any]]:
|
|
||||||
existing = jira.get_project_components(project_key)
|
|
||||||
|
|
||||||
needed_names: list[str] = []
|
|
||||||
for issue in issues:
|
|
||||||
issue_components = issue.components or [default_component]
|
|
||||||
for comp in issue_components:
|
|
||||||
key = normalize(comp)
|
|
||||||
if key and key not in existing and comp not in needed_names:
|
|
||||||
needed_names.append(comp)
|
|
||||||
|
|
||||||
if needed_names and not create_missing:
|
|
||||||
missing = ", ".join(needed_names)
|
|
||||||
raise RuntimeError(
|
|
||||||
"These components do not exist in Jira:\n"
|
|
||||||
f"{missing}\n\n"
|
|
||||||
"Re-run with --create-missing-components to create them automatically."
|
|
||||||
)
|
|
||||||
|
|
||||||
for name in needed_names:
|
|
||||||
created = jira.create_component(project_key, name)
|
|
||||||
existing[normalize(created["name"])] = created
|
|
||||||
print(f"Created component: {created['name']}")
|
|
||||||
|
|
||||||
return existing
|
|
||||||
|
|
||||||
|
|
||||||
def build_issue_payload(
|
|
||||||
issue: CsvIssue,
|
|
||||||
project_key: str,
|
|
||||||
issue_types_by_name: dict[str, dict[str, Any]],
|
|
||||||
create_fields_by_issue_type_id: dict[str, dict[str, Any]],
|
|
||||||
components_by_name: dict[str, dict[str, Any]],
|
|
||||||
default_component: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
work_type_key = normalize(issue.work_type)
|
|
||||||
if work_type_key not in issue_types_by_name:
|
|
||||||
known = ", ".join(sorted(it["name"] for it in issue_types_by_name.values()))
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Unknown Work Type '{issue.work_type}' for summary '{issue.summary}'. "
|
|
||||||
f"Known Jira issue types in project {project_key}: {known}"
|
|
||||||
)
|
|
||||||
|
|
||||||
issue_type = issue_types_by_name[work_type_key]
|
|
||||||
issue_type_id = issue_type["id"]
|
|
||||||
fields_meta = create_fields_by_issue_type_id[issue_type_id]
|
|
||||||
|
|
||||||
fields: dict[str, Any] = {
|
|
||||||
"project": {"key": project_key},
|
|
||||||
"summary": issue.summary,
|
|
||||||
"issuetype": {"id": issue_type_id},
|
|
||||||
}
|
|
||||||
|
|
||||||
if "description" in fields_meta:
|
|
||||||
fields["description"] = adf_paragraph(issue.description)
|
|
||||||
|
|
||||||
if issue.priority and "priority" in fields_meta:
|
|
||||||
fields["priority"] = {"name": issue.priority}
|
|
||||||
|
|
||||||
effective_components = issue.components or [default_component]
|
|
||||||
|
|
||||||
comps = []
|
|
||||||
for comp_name in effective_components:
|
|
||||||
key = normalize(comp_name)
|
|
||||||
component = components_by_name.get(key)
|
|
||||||
if not component:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Component '{comp_name}' not found for issue '{issue.summary}'"
|
|
||||||
)
|
|
||||||
comps.append({"id": component["id"]})
|
|
||||||
fields["components"] = comps
|
|
||||||
|
|
||||||
return {"fields": fields}
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
args = parse_args()
|
|
||||||
|
|
||||||
issues = read_csv_issues(args.csv)
|
|
||||||
if not issues:
|
|
||||||
print("No issues found in CSV.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
jira = JiraClient(
|
|
||||||
site=args.site,
|
|
||||||
email=args.email,
|
|
||||||
api_token=args.api_token,
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"Loaded {len(issues)} issues from CSV")
|
|
||||||
|
|
||||||
issue_types_by_name = jira.get_issue_types_for_project(args.project)
|
|
||||||
if not issue_types_by_name:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"No creatable issue types returned for project {args.project}. "
|
|
||||||
"Check permissions and project key."
|
|
||||||
)
|
|
||||||
|
|
||||||
needed_issue_type_ids = {
|
|
||||||
issue_types_by_name[normalize(issue.work_type)]["id"]
|
|
||||||
for issue in issues
|
|
||||||
if normalize(issue.work_type) in issue_types_by_name
|
|
||||||
}
|
|
||||||
|
|
||||||
create_fields_by_issue_type_id: dict[str, dict[str, Any]] = {}
|
|
||||||
for issue_type_id in needed_issue_type_ids:
|
|
||||||
create_fields_by_issue_type_id[issue_type_id] = jira.get_create_fields_metadata(
|
|
||||||
args.project,
|
|
||||||
issue_type_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
components_by_name = ensure_components(
|
|
||||||
jira=jira,
|
|
||||||
project_key=args.project,
|
|
||||||
issues=issues,
|
|
||||||
create_missing=args.create_missing_components,
|
|
||||||
default_component=args.default_component,
|
|
||||||
)
|
|
||||||
|
|
||||||
payloads: list[dict[str, Any]] = []
|
|
||||||
for issue in issues:
|
|
||||||
payloads.append(
|
|
||||||
build_issue_payload(
|
|
||||||
issue=issue,
|
|
||||||
project_key=args.project,
|
|
||||||
issue_types_by_name=issue_types_by_name,
|
|
||||||
create_fields_by_issue_type_id=create_fields_by_issue_type_id,
|
|
||||||
components_by_name=components_by_name,
|
|
||||||
default_component=args.default_component,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if args.dry_run:
|
|
||||||
print(json.dumps({"issueUpdates": payloads[:2]}, indent=2, ensure_ascii=False))
|
|
||||||
print(f"\nDry run only. Total issues prepared: {len(payloads)}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
created_total = 0
|
|
||||||
error_total = 0
|
|
||||||
|
|
||||||
for batch_no, batch in enumerate(chunked(payloads, 50), start=1):
|
|
||||||
print(f"Sending batch {batch_no} with {len(batch)} issues...")
|
|
||||||
result = jira.bulk_create_issues(batch)
|
|
||||||
|
|
||||||
created = result.get("issues", []) or []
|
|
||||||
errors = result.get("errors", []) or []
|
|
||||||
|
|
||||||
for item in created:
|
|
||||||
key = item.get("key", "<unknown>")
|
|
||||||
issue_id = item.get("id", "<unknown>")
|
|
||||||
print(f"Created: {key} (id={issue_id})")
|
|
||||||
|
|
||||||
for item in errors:
|
|
||||||
failed = item.get("failedElementNumber")
|
|
||||||
element_errors = item.get("elementErrors", {})
|
|
||||||
print(
|
|
||||||
f"ERROR in batch item index {failed}: "
|
|
||||||
f"{json.dumps(element_errors, ensure_ascii=False)}",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
|
|
||||||
created_total += len(created)
|
|
||||||
error_total += len(errors)
|
|
||||||
|
|
||||||
time.sleep(0.5)
|
|
||||||
|
|
||||||
print()
|
|
||||||
print(f"Done. Created: {created_total}, Errors: {error_total}")
|
|
||||||
|
|
||||||
return 0 if error_total == 0 else 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
try:
|
|
||||||
raise SystemExit(main())
|
|
||||||
except KeyboardInterrupt:
|
|
||||||
print("\nInterrupted.", file=sys.stderr)
|
|
||||||
raise SystemExit(130)
|
|
||||||
except Exception as exc:
|
|
||||||
print(f"\nFATAL: {exc}", file=sys.stderr)
|
|
||||||
raise SystemExit(1)
|
|
||||||
Loading…
Reference in New Issue