{% extends "base.html" %}
{% block title %}vq · fleet · {{ jobid }}{% endblock %}
{% block content %}
{% if error %}
<h2>
  <a href="/fleet/jobs">fleet jobs</a> ·
  <span class="jobid">{{ jobid }}</span>
</h2>
<p class="host-card-error">
  Could not read job {{ jobid }} on {{ queue_host }}: {{ error }}
</p>
<p class="hint"><a href="">Retry</a></p>
{% else %}
{% set s = detail.status %}
<h2>
  <a href="/fleet/jobs">fleet jobs</a> ·
  <span class="jobid">{{ jobid }}</span>
  <span class="state state-{{ s.state }}">{{ s.state }}</span>
  <span class="version">on {{ queue_host }}{% if detail.owner_host != queue_host %} (read on {{ detail.owner_host }}){% endif %}</span>
</h2>
<p class="hint">Fetched live from the owning host just now. <a href="/fleet/jobs/{{ jobid }}?host={{ queue_host }}">Refresh</a></p>

{% if flash %}
<p class="hint {% if not flash.startswith('ok') %}host-card-error{% endif %}"><strong>{{ flash }}</strong></p>
{% endif %}

{% if can_act and s.state in ("pending", "running", "suspended") %}
<section class="toolbar danger-zone" aria-label="job actions">
  {% if s.state in ("pending", "running") %}
  <form method="post" action="/fleet/jobs/{{ jobid }}/kill"
        onsubmit="return confirm('Kill {{ jobid }} on {{ queue_host }}?')">
    <input type="hidden" name="host" value="{{ queue_host }}">
    <label>reason <input type="text" name="reason" size="18"></label>
    <button type="submit">Kill</button>
  </form>
  {% endif %}
  {% if s.state == "running" %}
  <form method="post" action="/fleet/jobs/{{ jobid }}/pause">
    <input type="hidden" name="host" value="{{ queue_host }}">
    <button type="submit">Pause</button>
  </form>
  {% endif %}
  {% if s.state == "suspended" %}
  <form method="post" action="/fleet/jobs/{{ jobid }}/resume">
    <input type="hidden" name="host" value="{{ queue_host }}">
    <button type="submit">Resume</button>
  </form>
  {% endif %}
</section>
{% endif %}

<section class="spec">
  <h3>spec</h3>
  <dl>
    <dt>command</dt>           <dd><code>{{ s.command | join(" ") }}</code></dd>
    <dt>cwd</dt>               <dd><code>{{ s.cwd }}</code></dd>
    <dt>cpus</dt>              <dd>{{ s.cpus }}</dd>
    <dt>mem_mb</dt>            <dd>{{ s.mem_mb or "(undeclared)" }}</dd>
    <dt>wall_time_seconds</dt> <dd>{{ s.wall_time_seconds or "(undeclared)" }}</dd>
    <dt>submitter</dt>         <dd>{{ s.submitter or "-" }}</dd>
    <dt>job_name</dt>          <dd>{{ s.job_name or "-" }}</dd>
    {% if s.tags %}
    <dt>tags</dt>              <dd>{{ s.tags | join(", ") }}</dd>
    {% endif %}
    <dt>submitted</dt>         <dd>{{ s.submitted_at }}</dd>
    {% if s.started_at %}
    <dt>started</dt>           <dd>{{ s.started_at }}</dd>
    {% endif %}
    {% if s.finished_at %}
    <dt>finished</dt>          <dd>{{ s.finished_at }}</dd>
    {% endif %}
    {% if s.exit_code is not none %}
    <dt>exit_code</dt>         <dd>{{ s.exit_code | describe_exit }}</dd>
    {% endif %}
    {% if s.failure_reason %}
    <dt>failure_reason</dt>    <dd>{{ s.failure_reason }}</dd>
    {% endif %}
  </dl>
</section>

{% if s.scheduler_target %}
<section class="monitor">
  <h3>scheduler</h3>
  <dl>
    <dt>target</dt>       <dd>{{ s.scheduler_target }}</dd>
    {% if s.scheduler_id %}
    <dt>scheduler_id</dt> <dd><code>{{ s.scheduler_id }}</code></dd>
    {% endif %}
    {% if s.pbs_state_label %}
    <dt>scheduler state</dt> <dd>{{ s.pbs_state_label }}</dd>
    {% endif %}
    {% if s.fetch_state_label %}
    <dt>fetch state</dt>  <dd>{{ s.fetch_state_label }}</dd>
    {% endif %}
  </dl>
</section>
{% endif %}

{# #168: where the scheduler placed the job, node by node. #}
{% if allocation %}
<section class="monitor">
  <h3>nodes</h3>
  <table class="queue">
    <thead><tr><th>node</th><th>allocated cores</th></tr></thead>
    <tbody>
    {% for a in allocation %}
      <tr><td class="who">{{ a.node }}</td><td class="num">{{ a.cores if a.cores is not none else "not reported" }}</td></tr>
    {% endfor %}
    </tbody>
  </table>
  <p class="hint">The scheduler's allocation: {{ allocation | length }} node{{ "s" if allocation | length != 1 else "" }}. Nothing is measured per node; the usage on this page is the whole job's.</p>
</section>
{% endif %}

{# #168: what the finished command measurably used, from its receipt. #}
{% if final_usage %}
<section class="monitor">
  <h3>final resource usage</h3>
  {% if final_usage.status == "ok" %}
  <dl>
    <dt>wall</dt><dd>{{ final_usage.wall }}</dd>
    <dt>cpu time</dt><dd>{{ final_usage.cpu }}{% if final_usage.average_cores is not none %} ({{ final_usage.average_cores }} cores on average{% if final_usage.declared_cpus %} of {{ final_usage.declared_cpus }} declared{% endif %}){% endif %}</dd>
    <dt>peak rss</dt><dd>{% if final_usage.peak_rss_mb is not none %}{{ "%.0f" | format(final_usage.peak_rss_mb) }} MB{% if final_usage.peak_share is not none %} ({{ final_usage.peak_share }}% of {{ final_usage.declared_mem_mb }} MB declared){% endif %}{% else %}unknown{% endif %}</dd>
    {% if final_usage.process_count is not none %}<dt>processes</dt><dd>{{ final_usage.process_count }} at most at once</dd>{% endif %}
  </dl>
  <p class="hint">
    Measured by {{ final_usage.collector }}{% if final_usage.receipts and final_usage.receipts > 1 %} over {{ final_usage.receipts }} array elements (cpu summed, wall and peak the largest){% endif %}.
    Peak rss is the {{ final_usage.peak_rss_scope }}, not a sum over the job's processes.
  </p>
  {% elif final_usage.status == "absent" %}
  <p class="hint">No usage receipt. The command did not exit on its own (a killed job leaves none), or it ran under a vq that did not write one.</p>
  {% elif final_usage.status == "error" %}
  <p class="hint">The collector could not measure this run: {{ final_usage.error }}</p>
  {% else %}
  <p class="host-card-error">The usage receipt is not shown because it cannot be trusted: {{ final_usage.error }}</p>
  {% endif %}
</section>
{% endif %}

{# #168: a running GPU job's newest nvidia-smi sample. #}
{% if gpu_live %}
<section class="monitor">
  <h3>gpu now</h3>
  {% if gpu_live.status == "ok" %}
  <table class="queue">
    <thead><tr><th>gpu</th><th>model</th><th>utilisation</th><th>device memory</th><th>job processes</th></tr></thead>
    <tbody>
    {% for device in gpu_live.devices %}
    <tr>
      <td>{{ device.index }}</td><td>{{ device.name }}</td>
      <td>{% if device.utilization_percent is not none %}{{ device.utilization_percent }}%{% else %}unknown{% endif %}</td>
      <td>{% if device.memory_used_mb is not none %}{{ "%.0f" | format(device.memory_used_mb) }} MB{% else %}unknown{% endif %}{% if device.memory_total_mb is not none %} of {{ "%.0f" | format(device.memory_total_mb) }} MB{% endif %}</td>
      <td>{% if not device.job_process %}none seen{% elif device.job_memory_mb is not none %}{{ "%.0f" | format(device.job_memory_mb) }} MB{% else %}unknown{% endif %}</td>
    </tr>
    {% endfor %}
    </tbody>
  </table>
  <p class="hint">
    The newest complete nvidia-smi sample from the batch node, taken {{ gpu_live.observed or "at an unknown time" }} by the node's clock.
    {% if gpu_live.coverage == "allocation" %}The node shows exactly the {{ gpu_live.allocated }} allocated GPUs.
    {% elif gpu_live.coverage == "attributed" %}The node shows {{ gpu_live.visible }} GPUs for an allocation of {{ gpu_live.allocated }}; only devices a process of this job is on are listed.
    {% else %}The node shows {{ gpu_live.visible }} of {{ gpu_live.allocated }} allocated GPUs; other nodes are not observed.{% endif %}
    Utilisation and device memory are the device as a whole, not this job's share.
  </p>
  {% else %}
  <p class="hint">GPU use is not measured now: {{ gpu_live.reason }}</p>
  {% endif %}
</section>
{% endif %}

{# #168: what the job's GPUs did, from its nvidia-smi samples. #}
{% if gpu_usage %}
<section class="monitor">
  <h3>gpu use</h3>
  {% if gpu_usage.status == "ok" %}
  <table class="queue">
    <thead><tr>{% if gpu_usage.elements %}<th>element</th>{% endif %}<th>gpu</th><th>model</th><th>job processes peak</th><th>device memory peak</th><th>utilisation mean</th><th>utilisation peak</th></tr></thead>
    <tbody>
    {% for device in gpu_usage.devices %}
    <tr>
      {% if gpu_usage.elements %}<td>{{ device.element }}</td>{% endif %}<td>{{ device.index }}</td><td>{{ device.name }}</td>
      <td>{% if device.job_peak_memory_mb is not none %}{{ "%.0f" | format(device.job_peak_memory_mb) }} MB{% else %}not attributed{% endif %}</td>
      <td>{% if device.device_peak_memory_mb is not none %}{{ "%.0f" | format(device.device_peak_memory_mb) }} MB{% else %}unknown{% endif %}{% if device.memory_total_mb is not none %} of {{ "%.0f" | format(device.memory_total_mb) }} MB{% endif %}</td>
      <td>{% if device.utilization_mean_percent is not none %}{{ device.utilization_mean_percent }}%{% else %}unknown{% endif %}</td>
      <td>{% if device.utilization_peak_percent is not none %}{{ device.utilization_peak_percent }}%{% else %}unknown{% endif %}</td>
    </tr>
    {% endfor %}
    </tbody>
  </table>
  <p class="hint">
    Measured by {{ gpu_usage.collector }} on the batch node, {{ gpu_usage.samples }} samples every {{ gpu_usage.interval_seconds }} s{% if gpu_usage.truncated %}, until sampling reached its size cap{% endif %}.
    {% if gpu_usage.coverage == "allocation" %}The node showed exactly the {{ gpu_usage.allocated }} allocated GPUs, so these are the job's devices.
    {% elif gpu_usage.coverage == "attributed" %}The node showed {{ gpu_usage.visible }} GPUs for an allocation of {{ gpu_usage.allocated }}; only devices a process of this job was seen on are listed.
    {% elif gpu_usage.coverage == "mixed" %}Each of the {{ gpu_usage.elements | length }} array elements is summarised on its own node, and which devices are listed differs between them: {% for element in gpu_usage.elements %}element {{ element.index }} {{ element.coverage or ("not measured: " ~ element.reason) }}{% if not loop.last %}; {% endif %}{% endfor %}.
    {% else %}The node showed {{ gpu_usage.visible }} of {{ gpu_usage.allocated }} allocated GPUs; other nodes are not observed, so this is partial.{% endif %}
    Device memory and utilisation are the device as a whole, not this job's share. Job process memory counts processes started from the job script and, on Slurm, the job's other steps in its cgroup.
  </p>
  {% elif gpu_usage.status == "absent" %}
  <p class="hint">No GPU samples: the command never ran, or it ran under a vq that does not sample GPUs.</p>
  {% elif gpu_usage.status == "unavailable" %}
  <p class="hint">GPU use was not measured: {{ gpu_usage.reason }}</p>
  {% else %}
  <p class="host-card-error">The GPU samples are not shown because they cannot be trusted: {{ gpu_usage.error }}</p>
  {% endif %}
</section>
{% endif %}

{% if detail.samples %}
<section class="monitor">
  <h3>resource samples (last {{ detail.samples | length }}, newest first)</h3>
  <p class="hint">
    Watchdog samples from the owning host, ~5 s apart. CPU% is the
    whole process group ({{ s.cpus }} core(s) declared ⇒ up to ~{{ s.cpus * 100 }}%).
    {% if s.mem_mb %}Declared memory ceiling {{ s.mem_mb }} MB.{% endif %}
  </p>
  {# Four narrow columns, but a 19-char timestamp plus three numerics still
     exceeds a phone viewport once main's padding is taken off. #}
  <div class="table-scroll">
  <table class="events">
    <thead><tr><th>ts</th><th>cpu %</th><th>rss MB</th>{% if s.mem_mb %}<th>mem %</th>{% endif %}</tr></thead>
    <tbody>
      {% for smp in detail.samples | reverse %}
      <tr>
        <td class="ts">{{ smp.ts[:19] | replace("T", " ") if smp.ts else "-" }}</td>
        <td class="num">{{ "%.0f" | format(smp.cpu_percent) if smp.cpu_percent is not none else "-" }}</td>
        <td class="num">{{ "%.0f" | format(smp.rss_mb) if smp.rss_mb is not none else "-" }}</td>
        {% if s.mem_mb %}
        <td class="num">{{ "%.1f" | format(100.0 * smp.rss_mb / s.mem_mb) if smp.rss_mb is not none else "-" }}</td>
        {% endif %}
      </tr>
      {% endfor %}
    </tbody>
  </table>
  </div>
</section>
{% endif %}

{% if preview and (preview.method or preview.formula or preview.name or preview.scf
    or preview.error or preview.optimisation or preview.headline.total_energy is not none) %}
<section class="preview">
  <h3>calculation</h3>
  <dl>
    {% if preview.name or preview.formula %}<dt>system</dt><dd>{{ preview.name or "" }}{% if preview.name and preview.formula %} &middot; {% endif %}{{ preview.formula or "" }}{% if preview.atoms %} ({{ preview.atoms }} atoms{% if preview.periodic_dimensions %}, {{ preview.periodic_dimensions }}D periodic{% endif %}){% endif %}</dd>{% endif %}
    {% if preview.method %}<dt>method</dt><dd>{{ preview.method }}{% if preview.basis %} / {{ preview.basis }}{% endif %}</dd>{% endif %}
    {% if preview.headline.total_energy is not none %}<dt>total energy</dt><dd class="num">{{ "%.10f" | format(preview.headline.total_energy) }} Ha</dd>{% endif %}
    {% if preview.headline.gap_ev is not none %}<dt>gap</dt><dd class="num">{{ "%.4f" | format(preview.headline.gap_ev) }} eV</dd>{% endif %}
    {% if preview.scf %}<dt>SCF</dt><dd class="num">iteration {{ preview.scf[-1].iter }}, E = {{ "%.10f" | format(preview.scf[-1].energy) }} Ha, residual {{ "%.2e" | format(preview.scf[-1].residual) }}</dd>{% endif %}
    {% if preview.scf_energies | length > 1 %}<dt>SCF solutions</dt><dd class="num">{{ preview.scf_energies | length }} (latest {{ "%.10f" | format(preview.scf_energies[-1]) }} Ha)</dd>{% endif %}
    {% if preview.optimisation %}<dt>optimisation</dt><dd class="num">{{ preview.optimisation.iterations }} iterations, max|grad| {{ "%.2e" | format(preview.optimisation.max_gradient) }}, {{ "converged" if preview.optimisation.converged else "not converged" }}</dd>{% endif %}
    {% if preview.error %}<dt>error</dt><dd>{{ preview.error }}</dd>{% endif %}
  </dl>
  {% if chart %}
  <figure>
    <svg viewBox="0 0 {{ chart.width }} {{ chart.height }}" width="{{ chart.width }}" height="{{ chart.height }}" role="img" aria-label="SCF residual per iteration, log scale">
      <polyline fill="none" stroke="currentColor" stroke-width="1.5" points="{{ chart.points }}"/>
    </svg>
    <figcaption class="hint">log10 ||[F,DS]|| from {{ chart.log_top }} to {{ chart.log_bottom }}, iterations {{ chart.first_iter }}&ndash;{{ chart.last_iter }}</figcaption>
  </figure>
  {% endif %}
</section>
{% endif %}

<section class="logs">
  <h3>stdout (tail)</h3>
  {% if s.stdout %}<pre>{{ s.stdout }}</pre>{% else %}<p class="empty">No stdout captured yet.</p>{% endif %}
  <h3>stderr (tail)</h3>
  {% if s.stderr %}<pre>{{ s.stderr }}</pre>{% else %}<p class="empty">No stderr captured yet.</p>{% endif %}
</section>

<section class="events">
  <h3>events ({{ detail.events | length }})</h3>
  {% if detail.events %}
  {# Same unbounded detail cell as job_detail.html. #}
  <div class="table-scroll">
  <table class="events">
    <thead><tr><th>ts</th><th>kind</th><th>detail</th></tr></thead>
    <tbody>
      {% for e in detail.events %}
      <tr>
        <td class="ts">{{ e.ts }}</td>
        <td class="kind">{{ e.kind }}</td>
        <td class="detail">
          {%- set core = ["ts", "kind", "jobid"] -%}
          {%- for k, v in e.items() if k not in core -%}
            <strong>{{ k }}</strong>: <code>{{ v }}</code>{% if not loop.last %} · {% endif %}
          {%- endfor -%}
        </td>
      </tr>
      {% endfor %}
    </tbody>
  </table>
  </div>
  {% else %}
  <p class="empty">No events recorded (pre-v0.6.54 job, or workspace cleaned up).</p>
  {% endif %}
</section>
{% endif %}
{% endblock %}