→ Applies to: SynetoOS 6.x

This guide explains how to identify and reclaim disk space on the root filesystem (/) of a SynetoOS 6.x appliance. Follow these steps when disk usage is abnormally high or when you receive alerts about low available space.

Step 1. Connect to SynetoOS appliance via SSH as admin

ssh admin@<your_ip_address_or_hostname>

Step 2. Get root privileges

sudo su -

Step 3. Check the space usage of the root filesystem

df -h /

EXAMPLE OUTPUT

[root@syneto-os6-02 /]# df -h /
Filesystem      Size  Used  Avail  Use%  Mounted on
/dev/md126      196G   159G   37G   82%  /

If usage is above 70–80%, proceed to identify the cause.

Identify the Largest Directories

Find the largest directories under var

cd /var && du -csh *

EXAMPLE OUTPUT

[root@syneto-os6-02 var]# du -csh *
0       account
0       adm
3.5M    cache
44G     crash        ← kernel crash dumps
0       db
0       empty
0       ftp
0       games
0       kerberos
8.2G    lib
0       local
0       lock
38G     log          ← log files
0       mail
0       nis
0       opt
0       preserve
0       run
1.2G    spool
387M    storage
8.0K    svc
55M     syneto-edge
0       target
4.0K    tmp
204M    www
0       yp
147G    total

IMPORTANT
If /var/crash is consuming excessive space, follow the steps in the  Clean Up /var/crash section.
If /var/log is consuming excessive space, follow the steps in the Clean Up /var/log/esxi/ section.
If /var/lib is consuming excessive space, follow the steps in the Clean Up /var/lib section.

Clean Up /var/crash

Remove all dump older than 30 days

find /var/crash/ -type d -mtime +30 -exec rm -rf {} +

Crash dumps are large kernel core files generated after a system panic. They are safe to delete once the incident has been investigated. 
This command removes all dump directories older than 30 days.

Clean Up /var/log/esxi

Remove individual log files older than 30 days

find /var/log/esxi/ -type f -mtime +30 -delete

ESXi log forwarding is the most common cause of log directory bloat. This command removes individual log files older than 30 days under /var/log/esxi/.

Clean Up /var/lib

Step 1. Edit /tmp/clean_jobs.py file

vi /tmp/clean_jobs.py

IMPORTANT
Make sure to copy and paste the exact lines below.

#!/usr/bin/env python3
import asyncio
import os
import subprocess
import sys
from collections.abc import Iterator
from datetime import datetime, timezone
from pathlib import Path


def configure_output_buffering() -> None:
    """Flush output line-by-line when stdout is not attached to a TTY."""
    for stream in (sys.stdout, sys.stderr):
        if hasattr(stream, "reconfigure"):
            stream.reconfigure(line_buffering=True, write_through=True)


def _iter_exception_chain(exc: BaseException) -> Iterator[BaseException]:
    """Yield an exception and the exceptions that caused it."""
    current = exc
    while current is not None:
        yield current
        current = current.__cause__ or current.__context__


def _is_database_connection_closed_error(exc: BaseException) -> bool:
    """Return True when Tortoise/asyncpg lost the DB connection mid-cleanup."""
    connection_closed_messages = [
        "connection was closed in the middle of operation",
        "the underlying connection is closed",
        "'TransactionWrapper' object has no attribute '_template'",
    ]

    for current in _iter_exception_chain(exc):
        message = str(current)
        if any(error_message in message for error_message in connection_closed_messages):
            return True

    return False


def _print_database_retry_message() -> None:
    """Ask the user to retry without showing a scary traceback."""
    print(
        "The database connection was interrupted while cleaning jobs. "
        "Please re-run the script; it should continue normally.",
        file=sys.stderr,
    )


class CliAppState:
    pass


class CliApp:
    def __init__(self) -> None:
        self.state = CliAppState()

    def exception_handler(self, func):
        def wrapper(*args, **kwargs):
            return func(*args, **kwargs)

        return wrapper


class JobRecord:
    def __init__(self, job) -> None:
        self.id = job.id
        self._correlation_id = job.config["correlation_id"]
        self.created_at = job.created_at

        if job.type in ["vm_protect", "protect_local_vm", "protect_external_vm"]:
            self.snapshot_name = job.config["snapshot_name"]
            self.volume_id = job.config["storage"]["volume_id"]
            self.type = "vm_protect"
        elif job.type in ["volume_start_replication"]:
            self.snapshot_name = None
            self.volume_id = job.config["volume_id"]
            self.type = "volume_start_replication"
        else:
            raise ValueError(f"Unexpected job type: {job.type}")

    @property
    def correlation(self) -> str:
        return "-".join(self._correlation_id.split("-")[:-1])

    @property
    def is_protection(self) -> bool:
        return self.type == "vm_protect"


class SLAAggregate:
    def __init__(self) -> None:
        self._correlation = None
        self._created_at: datetime = None
        self._protections: list[JobRecord] = []
        self._replications: list[JobRecord] = []

    def add(self, rec: JobRecord):
        if self._correlation is None:
            self._correlation = rec.correlation
            self._created_at = rec.created_at
        elif self._correlation != rec.correlation:
            raise ValueError(
                f"Uncorrelated job={rec.id} job_correlation={rec.correlation} aggregate={self._correlation}"
            )
        if rec.volume_id and self.volume_ids and rec.volume_id not in self.volume_ids:
            raise ValueError(
                f"Uncorrelated volume job={rec.id} job_correlation={rec.correlation} aggregate={self._correlation}"
            )
        if rec.snapshot_name and self.snapshot_names and rec.snapshot_name not in self.snapshot_names:
            raise ValueError(
                f"Uncorrelated snapshot job={rec.id} job_correlation={rec.correlation} aggregate={self._correlation}"
            )

        self._created_at = min(self._created_at, rec.created_at)
        if rec.is_protection:
            self._protections.append(rec)
        else:
            self._replications.append(rec)

    @property
    def correlation(self) -> str:
        return self._correlation

    def is_expired(self) -> bool:
        age = datetime.now(timezone.utc) - self._created_at
        return age.days > 15

    @property
    def volume_ids(self) -> set:
        return {rec.volume_id for rec in [*self._protections, *self._replications] if rec.volume_id}

    @property
    def snapshot_names(self) -> set:
        return {rec.snapshot_name for rec in [*self._protections, *self._replications] if rec.snapshot_name}

    def get_job_ids(self) -> list[str]:
        return [str(job.id) for job in [*self._protections, *self._replications] if job.id]

    def volume_snapshot_key(self):
        volume_id = next((v for v in self.volume_ids), None)
        snapshot_name = next((s for s in self.snapshot_names), None)
        if volume_id and snapshot_name:
            return f"{volume_id}@{snapshot_name}"
        else:
            return None

    def __str__(self) -> str:
        return f"Agg correlation={self.correlation} protections={len(self._protections)} replications={len(self._replications)}"


def cluster_main():
    import tortoise
    from chronos.domain.repositories import PolicyAssignmentRepo, RecoveryPointRepo  # noqa
    import chronos.db as db
    from chronos.db import register_db
    from chronos.settings import CONFIG, getenv
    from syneto_api import Storage

    async def generate_jobs(**filters):
        more = True
        offset = 0
        limit = 1000
        while more:
            jobs = await db.Job.get_all(offset=offset, limit=limit, order="created_at", **filters)
            for job in jobs:
                try:
                    yield JobRecord(job)
                except Exception:
                    continue

            offset += len(jobs)
            more = len(jobs) > 0

    def get_all_snapshots() -> set[str]:
        api = Storage()
        api.set_async(False)
        result = set()
        for volume in api.get_volumes():
            volume_id = volume.get("id")
            result = {*result, *{f"{volume_id}@{snap['name']}" for snap in api.get_volume_snapshots(volume_id)}}
        return result

    async def delete_jobs(job_ids: list[str]):
        tasks_delete_query = """
            DELETE from jobs j 
            WHERE (j.parent_id = ANY($1)) or (j.id = ANY($1));
         """

        async with tortoise.transactions.in_transaction() as connection:
            await connection.execute_query(tasks_delete_query, [job_ids])

    async def main():
        default_db_url = CONFIG["CHRONOS"]["SERVICE_PERSISTENCE"]["DB_URI"]
        database_url = getenv("DATABASE_URL") or default_db_url
        print(f"Connecting database: {database_url}")

        job_types = ["vm_protect", "protect_local_vm", "protect_external_vm", "volume_start_replication"]
        try:
            app = CliApp()
            print("Starting job cleaning.")
            async with register_db(app):
                count = 0
                aggregates: dict[str, SLAAggregate] = {}
                async for job in generate_jobs(type__in=job_types):
                    aggregate = aggregates.get(job.correlation)
                    if aggregate is None:
                        aggregate = SLAAggregate()
                        aggregate.add(job)
                        aggregates[aggregate.correlation] = aggregate
                    else:
                        aggregate.add(job)

                    count += 1
                    if count % 1000 == 0:
                        print(f"Checked jobs={count} aggregates={len(aggregates)}")
                print(f"Checked jobs={count} aggregates={len(aggregates)}")

                snapshots = get_all_snapshots()

                surpluss: list[SLAAggregate] = []
                surpluss_jobs = 0

                retained = {}
                retained_jobs = 0

                non_keyed_retained = []
                non_keyed_retained_jobs = 0

                for agg in aggregates.values():
                    key = agg.volume_snapshot_key()
                    if key and key in snapshots:
                        retained[key] = agg
                        retained_jobs += len(agg._protections) + len(agg._replications)
                    elif not agg.is_expired():
                        non_keyed_retained.append(agg)
                        non_keyed_retained_jobs += len(agg._protections) + len(agg._replications)
                    else:
                        surpluss.append(agg)
                        surpluss_jobs += len(agg._protections) + len(agg._replications)

                print(f"Retaining snapshot tasks={len(retained)} records={retained_jobs}")
                print(f"Retaining recent non-snap tasks={len(non_keyed_retained)} records={non_keyed_retained_jobs}")
                print(f"Deleting surpluss tasks={len(surpluss)} records={surpluss_jobs}")

                batch = []
                deleted_count = 0
                for n, agg in enumerate(surpluss):
                    batch.extend(agg.get_job_ids())
                    if len(batch) > 150:
                        await delete_jobs(batch)
                        deleted_count += len(batch)
                        batch = []
                    if n % 1000 == 0:
                        print(f"Deleted jobs={deleted_count}/{surpluss_jobs}")
                await delete_jobs(batch)
                deleted_count += len(batch)
                print(f"Deleted jobs={deleted_count}/{surpluss_jobs}")

            print("Completed job cleaning.")
        finally:
            pass

    try:
        asyncio.run(main())
    except Exception as e:
        if _is_database_connection_closed_error(e):
            _print_database_retry_message()
            sys.exit(1)
        raise


def main():
    configure_output_buffering()

    if os.environ.get("KUBERNETES_SERVICE_HOST"):
        cluster_main()
        return

    script = Path(__file__).read_text(encoding="utf-8")
    result = subprocess.run(
        [
            "/usr/bin/kubecolor",
            "exec",
            "-i",
            "deployments/chronos",
            "--",
            "./.venv/bin/python3",
            "-u",
        ],
        input=script,
        text=True,
        check=False,
    )
    sys.exit(result.returncode)


if __name__ == "__main__":
    main()

Save and EXIT

:wq

Step 2. Give permissions to /tmp/clean_jobs.py file

chmod +x /tmp/clean_jobs.py

Step 3. Run the script

/tmp/clean_jobs.py

If the script exits with "The database connection was interrupted while cleaning jobs", simply re-run it — this is a known transient condition and the script is safe to restart.

Verify the Result

Check if the space has been reclaimed

df -h /

The Used% value should be noticeably lower. If usage remains high, repeat Step 2 to identify other large directories.