Migrate Large Volumes of Files from NAS and File Servers to the Cloud

import os
import requests

BASE_URL = os.getenv("INNORIX_BASE_URL", "https://app.innorix.com").rstrip("/")
TOKEN = os.environ["INNORIX_ACCESS_TOKEN"]
WORKSPACE_ID = os.getenv("INNORIX_WORKSPACE_ID")   # optional; falls back to the current workspace

STATUS_COMPLETE = 2
TERMINAL = {2, 4, 5, 9, 99}          # complete / error / cancelled / partial / failed
NOT_SUCCEEDED = {4, 5, 9, 99}


def api(method, path, body=None, params=None):
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {TOKEN}",
    }

    if WORKSPACE_ID:
        headers["x-workspace-id"] = WORKSPACE_ID

    response = requests.request(
        method, BASE_URL + path,
        headers=headers, json=body, params=params, timeout=30,
    )

    payload = response.json() if response.content else {}

    if not response.ok:
        raise RuntimeError(payload.get("message") or f"HTTP {response.status_code}")

    return payload.get("data")


def is_terminal(detail):
    return detail.get("isTerminal", detail.get("status") in TERMINAL)

import base64
import time


def encode_path(device_id, raw_path):
    normalized = str(raw_path or "").replace("\\", "/")
    token = base64.b64encode(normalized.encode("utf-8")).decode("ascii")
    return f"{device_id}_ino_{token}"


def now_iso():
    return time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime())

def scan(device_id, path, max_pages=200):
    page = api("POST", f"/api/devices/{device_id}/files/search",
               {"path": path, "pageSize": 1000})

    total = size = 0
    search_id = page.get("searchId")

    try:
        for _ in range(max_pages):
            for item in page.get("items") or []:
                if item["type"] == "file":
                    total += 1
                    size += item.get("size") or 0

            if not page.get("hasMore"):
                break

            page = api("GET", f"/api/devices/{device_id}/files/search",
                       params={"cursor": page["nextCursor"]})
    finally:
        # stop the device scan once the count is done
        if search_id:
            api("POST", f"/api/devices/{device_id}/files/search/cancel",
                {"uuid": search_id})

    return total, size


count, size = scan("device-nas-01", "/share/team")
print(f"{count} files / {size / (1024 ** 3):.1f}GB")

def target_for(folder, base):
    return folder.replace("/share", base, 1)


def migrate(source, target, folder, base):
    transfer = api("POST", "/api/transfers/manual", {
        "sourceDevice": source,
        "targetDevice": target,
        "targetPath": target_for(folder, base),
        "sourcePaths": [folder],
        "sendAllFolder": True,
        "transferOptions": {"target-action": "overwrite"},
    })

    return transfer["monitorId"]


FOLDERS = ["/share/team/design", "/share/team/docs", "/share/team/archive"]

transfers = {
    folder: migrate("device-nas-01", "device-cloud-01", folder, "/migrated")
    for folder in FOLDERS
}

def wait(monitor_id, timeout=3600, interval=3):
    deadline = time.time() + timeout

    while time.time() < deadline:
        detail = api("GET", f"/api/transfers/{monitor_id}")

        if is_terminal(detail):
            return detail

        time.sleep(interval)

    raise TimeoutError(monitor_id)


def failed_files(monitor_id):
    result = api("GET", f"/api/transfers/{monitor_id}/files", params={
        "state": "any", "size": 500,
    }) or {}

    return [r for r in (result.get("children") or [])
            if r.get("status") in NOT_SUCCEEDED]


def retry_failed(monitor_id):
    rows = failed_files(monitor_id)

    if not rows:
        return 0

    api("POST", f"/api/transfers/{monitor_id}/retry", {
        "filesRetry": [
            {"filePath": r["sourceFilePath"], "isDir": bool(r.get("isFolder"))}
            for r in rows
        ]
    })

    return len(rows)

for folder, monitor_id in transfers.items():
    detail = wait(monitor_id, timeout=86400)

    print(f"{folder:28} {detail.get('statusName')}"
          f" {detail.get('fileCount')} files")

    if detail["status"] != STATUS_COMPLETE:
        print(f"  retried {retry_failed(monitor_id)} files")

transfer = api("POST", "/api/transfers/manual", {
    "sourceDevice": "device-nas-01",
    "targetDevice": "device-cloud-01",
    "targetPath": "/migrated/team/design",
    "sourcePaths": ["/share/team/design"],
    "sendAllFolder": True,
    "incremental": True,
    "transferOptions": {"target-action": "overwrite"},
})

def build_sync(name, source, source_path, target, target_path, schedule):
    return {
        "name": name,
        "flowName": name,
        "transferType": "normal",
        "timezone": "Asia/Seoul",
        "step": 1,
        "isUpcoming": False,
        "details": [
            {
                "senderId": source,
                "receiverId": target,
                "sourceItem": [
                    {
                        "hash": encode_path(source, source_path),
                        "filePath": source_path,
                        "isDir": True,
                    }
                ],
                "targetPath": encode_path(target, target_path),
                "step": 1,
                "transferOptions": {
                    "noSchedule": False,
                    "target-action": "overwrite",
                    "send-fileoption": {},
                },
            }
        ],
        "schedules": [schedule],
    }


HOURLY = {
    "type": "hour",
    "startDateType": "now",
    "startDate": now_iso(),
    "timezone": "Asia/Seoul",
}

syncs = {
    folder: api("POST", "/api/automations", build_sync(
        f"sync {folder}", "device-nas-01", folder,
        "device-cloud-01", target_for(folder, "/migrated"),
        HOURLY))["automationId"]
    for folder in FOLDERS
}

def progress(transfers):
    for folder, monitor_id in transfers.items():
        detail = api("GET", f"/api/transfers/{monitor_id}")

        print(f"{folder:28} {detail.get('statusName'):12}"
              f" {detail.get('percent', 0):>5}%"
              f" {detail.get('fileCount', 0)} files")

for folder, automation_id in syncs.items():
    runs = api("GET", f"/api/automations/{automation_id}/executions") or []
    latest = runs[0] if runs else {}

    print(f"{folder:28} runs {len(runs):>3}"
          f" last {latest.get('startTime')} {latest.get('status')}")

def list_names(device_id, path):
    names, page = set(), 1

    while True:
        result = api("GET", f"/api/devices/{device_id}/files", params={
            "path": path, "page": page, "size": 200, "type": "file",
        }) or {}

        names.update(item["name"] for item in result.get("items") or [])

        if page >= (result.get("lastPage") or 1):
            return names

        page += 1


missing = list_names("device-nas-01", "/share/team/design") \
        - list_names("device-cloud-01", "/migrated/team/design")

print(f"{len(missing)} missing")

for automation_id in syncs.values():
    api("POST", f"/api/automations/{automation_id}/pause", {"pause": True})

for automation_id in syncs.values():
    api("DELETE", f"/api/automations/{automation_id}")