{
 "$schema": "https://totktonada.online/tarantool/ci/causes.schema.json",
 "page": "https://totktonada.online/tarantool/ci/causes",
 "crawled": "2026-10-26T21:30:54+03:00",
 "generated": "2026-10-26T22:06:13+03:00",
 "scope": {
  "failed_jobs": 1053,
  "counted": 328,
  "on_master_or_release": 231,
  "passed_on_rerun": 97,
  "left_out": 725
 },
 "weeks": {
  "spans": [
   {
    "label": "Oct 5-12",
    "from": "2026-10-05T21:30:54+03:00",
    "to": "2026-10-12T21:30:54+03:00",
    "failed": 39,
    "jobs_run": 5451
   },
   {
    "label": "Oct 12-19",
    "from": "2026-10-12T21:30:54+03:00",
    "to": "2026-10-19T21:30:54+03:00",
    "failed": 174,
    "jobs_run": 8296
   },
   {
    "label": "Oct 19-26",
    "from": "2026-10-19T21:30:54+03:00",
    "to": "2026-10-26T21:30:54+03:00",
    "failed": 115,
    "jobs_run": 7014
   }
  ],
  "causes": [
   {
    "cause": "module or connector tests failed",
    "failed": [
     3,
     19,
     51
    ],
    "per_1000_jobs_run": [
     0.6,
     2.3,
     7.3
    ]
   },
   {
    "cause": "ctest timeout",
    "failed": [
     6,
     23,
     19
    ],
    "per_1000_jobs_run": [
     1.1,
     2.8,
     2.7
    ]
   },
   {
    "cause": "test failed",
    "failed": [
     15,
     9,
     17
    ],
    "per_1000_jobs_run": [
     2.8,
     1.1,
     2.4
    ]
   },
   {
    "cause": "VK S3 unreachable",
    "failed": [
     2,
     104,
     13
    ],
    "per_1000_jobs_run": [
     0.4,
     12.5,
     1.9
    ]
   },
   {
    "cause": "entrypoint tag missing",
    "failed": [
     0,
     0,
     8
    ],
    "per_1000_jobs_run": [
     0.0,
     0.0,
     1.1
    ]
   },
   {
    "cause": "distro mirror",
    "failed": [
     5,
     29,
     4
    ],
    "per_1000_jobs_run": [
     0.9,
     3.5,
     0.6
    ]
   },
   {
    "cause": "pecl.php.net unreachable",
    "failed": [
     0,
     0,
     2
    ],
    "per_1000_jobs_run": [
     0.0,
     0.0,
     0.3
    ]
   },
   {
    "cause": "github.com unreachable",
    "failed": [
     0,
     1,
     1
    ],
    "per_1000_jobs_run": [
     0.0,
     0.1,
     0.1
    ]
   },
   {
    "cause": "branch not found",
    "failed": [
     0,
     0,
     1
    ],
    "per_1000_jobs_run": [
     0.0,
     0.0,
     0.1
    ]
   },
   {
    "cause": "build error",
    "failed": [
     0,
     19,
     0
    ],
    "per_1000_jobs_run": [
     0.0,
     2.3,
     0.0
    ]
   },
   {
    "cause": "coveralls.io unreachable",
    "failed": [
     9,
     0,
     0
    ],
    "per_1000_jobs_run": [
     1.7,
     0.0,
     0.0
    ]
   },
   {
    "cause": "unexplained",
    "failed": [
     1,
     3,
     0
    ],
    "per_1000_jobs_run": [
     0.2,
     0.4,
     0.0
    ]
   },
   {
    "cause": "git transfer dropped",
    "failed": [
     0,
     2,
     0
    ],
    "per_1000_jobs_run": [
     0.0,
     0.2,
     0.0
    ]
   },
   {
    "cause": "killed",
    "failed": [
     1,
     1,
     0
    ],
    "per_1000_jobs_run": [
     0.2,
     0.1,
     0.0
    ]
   },
   {
    "cause": "python traceback",
    "failed": [
     0,
     2,
     0
    ],
    "per_1000_jobs_run": [
     0.0,
     0.2,
     0.0
    ]
   },
   {
    "cause": "name resolution failed",
    "failed": [
     0,
     1,
     0
    ],
    "per_1000_jobs_run": [
     0.0,
     0.1,
     0.0
    ]
   }
  ]
 },
 "days": {
  "spans": [
   {
    "label": "Mon 19",
    "from": "2026-10-19T21:30:54+03:00",
    "to": "2026-10-20T21:30:54+03:00",
    "failed": 13,
    "jobs_run": 2325
   },
   {
    "label": "Tue 20",
    "from": "2026-10-20T21:30:54+03:00",
    "to": "2026-10-21T21:30:54+03:00",
    "failed": 48,
    "jobs_run": 1568
   },
   {
    "label": "Wed 21",
    "from": "2026-10-21T21:30:54+03:00",
    "to": "2026-10-22T21:30:54+03:00",
    "failed": 38,
    "jobs_run": 1021
   },
   {
    "label": "Thu 22",
    "from": "2026-10-22T21:30:54+03:00",
    "to": "2026-10-23T21:30:54+03:00",
    "failed": 14,
    "jobs_run": 1285
   },
   {
    "label": "Fri 23",
    "from": "2026-10-23T21:30:54+03:00",
    "to": "2026-10-24T21:30:54+03:00",
    "failed": 0,
    "jobs_run": 150
   },
   {
    "label": "Sat 24",
    "from": "2026-10-24T21:30:54+03:00",
    "to": "2026-10-25T21:30:54+03:00",
    "failed": 0,
    "jobs_run": 32
   },
   {
    "label": "Sun 25",
    "from": "2026-10-25T21:30:54+03:00",
    "to": "2026-10-26T21:30:54+03:00",
    "failed": 2,
    "jobs_run": 633
   }
  ],
  "causes": [
   {
    "cause": "ctest timeout",
    "failed": [
     6,
     2,
     6,
     3,
     0,
     0,
     2
    ],
    "per_1000_jobs_run": [
     2.6,
     1.3,
     5.9,
     2.3,
     0.0,
     0.0,
     3.2
    ]
   },
   {
    "cause": "module or connector tests failed",
    "failed": [
     3,
     26,
     20,
     2,
     0,
     0,
     0
    ],
    "per_1000_jobs_run": [
     1.3,
     16.6,
     19.6,
     1.6,
     0.0,
     0.0,
     0.0
    ]
   },
   {
    "cause": "test failed",
    "failed": [
     2,
     5,
     2,
     8,
     0,
     0,
     0
    ],
    "per_1000_jobs_run": [
     0.9,
     3.2,
     2.0,
     6.2,
     0.0,
     0.0,
     0.0
    ]
   },
   {
    "cause": "VK S3 unreachable",
    "failed": [
     1,
     9,
     2,
     1,
     0,
     0,
     0
    ],
    "per_1000_jobs_run": [
     0.4,
     5.7,
     2.0,
     0.8,
     0.0,
     0.0,
     0.0
    ]
   },
   {
    "cause": "entrypoint tag missing",
    "failed": [
     0,
     0,
     8,
     0,
     0,
     0,
     0
    ],
    "per_1000_jobs_run": [
     0.0,
     0.0,
     7.8,
     0.0,
     0.0,
     0.0,
     0.0
    ]
   },
   {
    "cause": "distro mirror",
    "failed": [
     1,
     3,
     0,
     0,
     0,
     0,
     0
    ],
    "per_1000_jobs_run": [
     0.4,
     1.9,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0
    ]
   },
   {
    "cause": "pecl.php.net unreachable",
    "failed": [
     0,
     2,
     0,
     0,
     0,
     0,
     0
    ],
    "per_1000_jobs_run": [
     0.0,
     1.3,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0
    ]
   },
   {
    "cause": "branch not found",
    "failed": [
     0,
     1,
     0,
     0,
     0,
     0,
     0
    ],
    "per_1000_jobs_run": [
     0.0,
     0.6,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0
    ]
   },
   {
    "cause": "github.com unreachable",
    "failed": [
     0,
     1,
     0,
     0,
     0,
     0,
     0
    ],
    "per_1000_jobs_run": [
     0.0,
     0.6,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0
    ]
   }
  ]
 },
 "classifiers": [
  {
   "cause": "runner disk",
   "code": "# One cause, several faces: the disk fills, apt's lists are truncated and\n# apt reports a signature it cannot verify; rpm and dnf ask for more space;\n# yum finds its download directory full.\n('runner disk', re.compile(\n    r'(At least one invalid signature was encountered|No space left on device'\n    r'|At least \\d+ ?[KMG]i?B more space needed on the \\S+ filesystem'\n    r'|needs \\d+ ?[KMG]i?B more space on the \\S+ filesystem'\n    r'|Insufficient space in download directory)', re.I)),"
  },
  {
   "cause": "test failed",
   "code": "# A test that failed at least once, which a retry may have undone, or the\n# final verdict of test-run or ctest, which nothing undoes. A module's\n# integration job runs the module's own, older test-run: no worker\n# number before a result, and '===== N tests failed:' as the verdict --\n# 5 vshard failures read as unexplained until 2026-09-24; the line is in\n# none of 3,237 passing logs. Other modules run other runners: crud's\n# luatest closes with 'Failed tests:', php-queue's PHPUnit with\n# 'FAILURES!' -- 50 failures read as unexplained until 2026-10-26, 42 of\n# them php-queue's stats tests against the new stats.extra of\n# tarantool/queue@f789e07c87e7 (10-20 to 10-22, fixed by\n# tarantool-php/queue#27); neither line is in 300 passing logs. In the\n# integration workflow all of these are the modules' and connectors'\n# tests (classify).\n('test failed', re.compile(\n    r'^\\[\\d+\\]\\s+\\S+.*\\[ fail \\]'\n    r'|^Failed tasks:'\n    r'|^Failed tests:$'\n    r'|^(?:FAILURES|ERRORS)!$'\n    r'|^===== \\d+ tests? failed:'\n    r'|^\\s+\\d+ - .+ \\((?:Failed|SEGFAULT|Subprocess aborted|Child aborted'\n    r'|OTHER_FAULT|ILLEGAL|INTERRUPT|NUMERICAL|BAD_COMMAND|Exception'\n    r'|Not Run)\\)(?:\\s.*)?$', re.M)),"
  },
  {
   "cause": "github.com unreachable",
   "code": "# GitHub's artifact service as well, which the packaging tests download\n# their test libraries from: one job of 2026-10-21, ECONNRESET.\n('github.com unreachable', re.compile(\n    r'(Failed to connect to (?:codeload\\.)?github\\.com'\n    r'|Could not resolve host: (?:codeload\\.)?github\\.com'\n    r'|Failed to download archive .https://codeload\\.github\\.com'\n    r'|Unable to download artifact\\(s\\): Failed to ListArtifacts: Unable to'\n    r' make request)', re.I)),"
  },
  {
   "cause": "git transfer dropped",
   "code": "# A git transfer that started and then broke off. Kept apart from\n# github.com unreachable until it is clear they are the same path failing:\n# here the connection was made. The TLS lines are git built with GnuTLS\n# (the packpack images) or OpenSSL reporting a reset or a mangled record\n# mid-stream: 12 jobs read as unexplained until 2026-09-28, 11 on vds*.\n# curl's own words for the same, \"Failure when receiving data from the\n# peer\" and \"Empty reply from server\": 2 jobs of 2026-10-14, until\n# 2026-10-26.\n('git transfer dropped', re.compile(\n    r'RPC failed; curl \\d+|unexpected disconnect while reading sideband packet'\n    r'|fatal: early EOF|invalid index-pack output'\n    r\"|fatal: unable to access '[^']*': (?:GnuTLS recv error\"\n    r'|OpenSSL SSL_(?:read|connect)|Recv failure'\n    r'|Failure when receiving data from the peer|Empty reply from server)'\n    r'|the remote end hung up upon initial contact')),"
  },
  {
   "cause": "submodule commit gone",
   "code": "# A submodule commit GitHub no longer has. `git fetch` recurses into\n# submodules on demand, so the environment action's `git fetch --tags -f`\n# in a workspace without the history -- fetch-depth 1 on a runner back\n# from a stop -- asks for every commit a submodule ever pointed at, and\n# luajit bd34ea57 (2015) is gone. Once per workspace; see todo.d/0021.\n('submodule commit gone', re.compile(\n    r'upload-pack: not our ref [0-9a-f]{40}')),"
  },
  {
   "cause": "ctest timeout",
   "code": "# ctest's closing summary, not its per-test '***Timeout' line: ctest runs\n# with --repeat until-pass:3, so a timeout that passed on a repeat leaves\n# that line in a green log (9 of 126 green ctest jobs). A newer ctest\n# prints the test's labels after the verdict (3.31 in the FreeBSD VM, 4.x\n# on macOS: \"(Timeout)          app-luatest regression\"); 3.17, the\n# static build's, does not.\n('ctest timeout', re.compile(r'^\\s+\\d+ - .+ \\(Timeout\\)(?:\\s.*)?$',\n                             re.M)),"
  },
  {
   "cause": "distro mirror",
   "code": "# yum's \"No more mirrors to try\" is the CentOS 7 build failing on a\n# package. apt's warning that it could not connect counts only with the\n# failure it leads to, a package missing from the index that stayed\n# behind: alone, it is in 7 jobs of 2026-10-07 that went on and failed\n# on a test or on luacheck. One job each, unexplained until 2026-10-26.\n('distro mirror', re.compile(\n    r'(E: Failed to fetch|Cannot (?:download|prepare internal) mirrorlist'\n    r'|Errors during downloading metadata'\n    r'|temporary error \\(try again later\\)'\n    r'|\\[Errno 256\\] No more mirrors to try'\n    r\"|W: Failed to fetch \\S+ +Connection failed \\[IP:[\\s\\S]*?\"\n    r\"E: (?:Package '[^']+' has no installation candidate\"\n    r'|Unable to locate package))', re.I)),"
  },
  {
   "cause": "luarocks",
   "code": "('luarocks', re.compile(\n    r'(Error: Failed downloading.{0,80}(?:luarocks|rocks\\.tarantool)'\n    r'|Could not resolve host: (?:www\\.)?luarocks\\.org'\n    r'|Failed to connect to (?:www\\.)?luarocks\\.org)', re.I)),"
  },
  {
   "cause": "coveralls.io unreachable",
   "code": "# coveralls-lcov's connect to coveralls.io (Cloudflare) outlived Ruby's\n# 60 s open_timeout. The gem retries a 5xx, not an exception, so one\n# dropped connect fails the coverage job. 3 jobs on 10-02 and 10-04,\n# unexplained until 2026-10-05; see todo.d/0022.\n('coveralls.io unreachable', re.compile(\n    r'\\(Net::OpenTimeout\\)(?:\\n\\s+from [^\\n]*)*?\\n\\s+from \\S*/coveralls-lcov-')),"
  },
  {
   "cause": "VK S3 unreachable",
   "code": "VK_S3 = (r'(?:hb\\.bizmrg\\.com|[\\w.-]*hb\\.vkcs\\.cloud|[\\w.-]*hb\\.vkcloud-storage\\.ru'\n         r'|95\\.163\\.53\\.117)')\n\n# Everything CI fetches from VK goes through one address, 95.163.53.117:\n# download.tarantool.org and rocks.tarantool.org redirect to hb.bizmrg.com,\n# and the packpack, distrib and multivac buckets are *.hb.vkcs.cloud and\n# *.hb.vkcloud-storage.ru names for it. Its front end dropped and stalled\n# connections on 2026-10-14 15:17-16:54 UTC and from 10-15 13:58 UTC; 57\n# failed jobs read as unexplained until then. Only what names the address\n# counts: apt merely warns about a repository it could not reach, and the\n# job then fails on a package or a key from it, which can have other\n# causes. See todo.d/0023. curl's \"Connection reset by peer in connection\n# to <host>\" and apt's \"Err:\" line, whose reason names the address on the\n# line after it, count too: one job each, until 2026-10-26.\n('VK S3 unreachable', re.compile(\n    r'(?:Could not connect to|Unable to connect to|Failed to connect to'\n    r'|connect to|in connection to|ETIMEDOUT'\n    r'|Timeout was reached for https://)\\s*' + VK_S3\n    + r'|' + VK_S3 + r'\\S*: \\[Errno \\d+\\] Timeout on'\n    r'|Failed to fetch https://(?:download\\.tarantool\\.org|' + VK_S3\n    + r')\\S* +Could not wait for server fd'\n    r'|Could not wait for server fd[^\\n]*\\[IP: ' + VK_S3\n    + r'|Failed downloading https?://rocks\\.tarantool\\.org')),"
  },
  {
   "cause": "checkpatch style",
   "code": "('checkpatch style', re.compile(r'has style problems, please review')),"
  },
  {
   "cause": "luacheck",
   "code": "('luacheck', re.compile(\n    r'^Total: (?:[1-9]\\d* warnings? / \\d+ errors?'\n    r'|\\d+ warnings? / [1-9]\\d* errors?) in \\d+ files', re.M)),"
  },
  {
   "cause": "build error",
   "code": "# The compiler, the linker, CMake or ninja said so. Not make's own\n# \"*** [target] Error\": a test, luacheck or packaging target fails through\n# make too, and that line made every ctest failure a build error.\n('build error', re.compile(\n    r'CMake Error'\n    r'|^\\S+:\\d+(?::\\d+)?: (?:fatal )?error: '\n    r'|error: .*\\[-W'\n    r'|undefined reference to |collect2: error: ld returned'\n    r'|ld(?:\\.lld|\\.gold)?: error:|linker command failed'\n    r'|No rule to make target'\n    r'|^FAILED: |^ninja: build stopped', re.M)),"
  },
  {
   "cause": "apk version",
   "code": "# abuild rejects the version of a 2.11 entrypoint commit; see\n# ~/w/alpine-pkg-version.\n('apk version', re.compile(r'ERROR: \\S+: \\S+ is not a valid version')),"
  },
  {
   "cause": "python traceback",
   "code": "('python traceback', re.compile(r'^Traceback \\(most recent call last\\)', re.M)),"
  },
  {
   "cause": "OOM",
   "code": "('OOM', re.compile(r'Out of memory: Killed|oom-kill', re.I)),"
  },
  {
   "cause": "killed",
   "code": "# A step that ended on SIGKILL, 128 + 9, with no word of why: the OOM\n# killer of the runner's container leaves none in the job's log, and\n# neither does anything else that kills. 2 jobs on ghacts-shared runners,\n# 10-07 and 10-15, until 2026-10-26; in none of 300 passing logs.\n('killed', re.compile(r'##\\[error\\]Process completed with exit code 137\\.')),"
  },
  {
   "cause": "docker pull",
   "code": "('docker pull', re.compile(r'(Docker pull failed|toomanyrequests|manifest unknown)', re.I)),"
  },
  {
   "cause": "entrypoint tag missing",
   "code": "# tools/check-entrypoint-tag.sh: once a release branch is cut, master's\n# next commit needs the next entrypoint tag, and until it is pushed every\n# run of master fails here. 8 jobs on 2026-10-22, after release/3.9 --\n# two runs of four attempts each -- unexplained until 2026-10-26.\n('entrypoint tag missing', re.compile(\n    r'Missing tag for commit [0-9a-f]{40} after branching')),"
  },
  {
   "cause": "pecl.php.net unreachable",
   "code": "# The PHP connectors' images install extensions from PECL, whose REST\n# interface gave nothing back within 20-30 s: 2 jobs, 10-20 and 10-21.\n('pecl.php.net unreachable', re.compile(\n    r'No releases available for package \"pecl\\.php\\.net/'\n    r'|does not have REST info xml available')),"
  },
  {
   "cause": "name resolution failed",
   "code": "# A getaddrinfo failure, whoever asked: rubygems.org from gem install on\n# 2026-10-15, unexplained until 2026-10-26.\n('name resolution failed', re.compile(\n    r'Temporary failure in name resolution|Temporary failure resolving')),"
  },
  {
   "cause": "branch not found",
   "code": "# actions/checkout asked for a branch the repository does not have:\n# tarantool-ee's release/3.9, which came later than tarantool's.\n('branch not found', re.compile(\n    r\"A branch or tag with the name '[^']+' could not be found\")),"
  },
  {
   "cause": "module or connector tests failed",
   "code": "# The integration workflow runs the own tests of modules (vshard, crud,\n# metrics, ...) and of connectors (tarantool-python, php-client, php-queue,\n# go-tarantool, ...) against tarantool. A test failing there is counted\n# apart from tarantool's own tests, which are the ones the flaky ranking\n# names: it can be tarantool breaking the project, or the project itself --\n# php-queue's tests against a new queue were 42 failures in October 2026\n# (decided with the user, 2026-10-26).\nMODULES = 'integration'\nMODULE_TESTS = 'module or connector tests failed'\n\n\ndef classify(text, workflow):\n    \"\"\"The causes a failed job's log shows: every signature it matches, a\n    test failure of the modules' and connectors' workflow under a name of\n    its own.\"\"\"\n    signs = [name for name, rx in SIGNS if rx.search(text)]\n    if workflow == MODULES:\n        signs = [MODULE_TESTS if s == 'test failed' else s for s in signs]\n    return signs or ['unexplained']"
  }
 ]
}