Cockroach: sentry: conn_executor.go:670: panic while executing 1 statements: SELECT _._, _._, ((SELECT _._ FROM _ AS _ WHERE _._ = _._ LIMIT _) - IFNULL(_._, _)) AS _, _._, _._ FROM _ AS _ JOIN _ AS _ ON _._ = _._ WHERE _._ = _ ORDER BY _._: caused by <redacted>

[tbg]

https://sentry.io/cockroach-labs/cockroachdb/issues/747736774/

conn_executor.go:670: panic while executing 1 statements: SELECT _._, _._, ((SELECT _._ FROM _ AS _ WHERE _._ = _._ LIMIT _) - IFNULL(_._, _)) AS _, _._, _._ FROM _ AS _ JOIN _ AS _ ON _._ = _._ WHERE _._ = _ ORDER BY _._: caused by

github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn.func1

stacktrace: {u'frames': [{u'function': u'func4', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql/pgwire.(conn).serveImpl', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go', u'lineno': 309, u'in_app': True}, {u'function': u'ServeConn', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(Server)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'lineno': 420, u'in_app': True}, {u'function': u'run', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(connExecutor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'lineno': 1143, u'in_app': True}, {u'function': u'execStmt', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(connExecutor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go', u'lineno': 95, u'in_app': True}, {u'function': u'execStmtInOpenState', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(connExecutor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go', u'lineno': 403, u'in_app': True}, {u'function': u'dispatchToExecutionEngine', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(connExecutor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go', u'lineno': 825, u'in_app': True}, {u'function': u'execWithDistSQLEngine', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(connExecutor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go', u'lineno': 983, u'in_app': True}, {u'function': u'PlanAndRun', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go', u'lineno': 740, u'in_app': True}, {u'function': u'createPlanForNode', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 2279, u'in_app': True}, {u'function': u'createPlanForNode', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 2329, u'in_app': True}, {u'function': u'createPlanForDistinct', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 2658, u'in_app': True}, {u'function': u'createPlanForNode', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 2279, u'in_app': True}, {u'function': u'createPlanForNode', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 2273, u'in_app': True}, {u'function': u'createPlanForLookupJoin', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 1844, u'in_app': True}, {u'function': u'createPlanForNode', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 2366, u'in_app': True}, {u'function': u'wrapPlan', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 2412, u'in_app': True}, {u'function': u'walkPlan', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/walk.go', u'lineno': 65, u'in_app': True}, {u'function': u'visit', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(planVisitor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/walk.go', u'lineno': 101, u'in_app': True}, {u'function': u'visitInternal', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(planVisitor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/walk.go', u'lineno': 504, u'in_app': True}, {u'function': u'visit', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(planVisitor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/walk.go', u'lineno': 101, u'in_app': True}, {u'function': u'visitInternal', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(planVisitor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/walk.go', u'lineno': 124, u'in_app': True}, {u'function': u'func1', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner).wrapPlan', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 2437, u'in_app': True}, {u'function': u'createPlanForNode', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 2276, u'in_app': True}, {u'function': u'createPlanForJoin', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 1965, u'in_app': True}, {u'function': u'tryCreatePlanForInterleavedJoin', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_plan_join.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_plan_join.go', u'lineno': 120, u'in_app': True}, {u'function': u'PartitionSpans', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(DistSQLPlanner)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go', u'lineno': 699, u'in_app': True}, {u'function': u'sigpanic', u'abs_path': u'/usr/local/go/src/runtime/signal_unix.go', u'module': u'runtime', u'filename': u'runtime/signal_unix.go', u'lineno': 388, u'in_app': False}, {u'function': u'panicmem', u'abs_path': u'/usr/local/go/src/runtime/panic.go', u'module': u'runtime', u'filename': u'runtime/panic.go', u'lineno': 63, u'in_app': False}, {u'function': u'gopanic', u'abs_path': u'/usr/local/go/src/runtime/panic.go', u'module': u'runtime', u'filename': u'runtime/panic.go', u'lineno': 502, u'in_app': False}, {u'function': u'call32', u'abs_path': u'/usr/local/go/src/runtime/asm_amd64.s', u'module': u'runtime', u'filename': u'runtime/asm_amd64.s', u'lineno': 573, u'in_app': False}, {u'function': u'func1', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(Server).ServeConn', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'lineno': 418, u'in_app': True}]}
type: *log.safeError
value: conn_executor.go:670: panic while executing 1 statements: SELECT _._, _._, ((SELECT _._ FROM _ AS _ WHERE _._ = _._ LIMIT _) - IFNULL(_._, _)) AS _, _._, _._ FROM _ AS _ JOIN _ AS _ ON _._ = _._ WHERE _._ = _ ORDER BY _._: caused by

[bithavoc]

holy shit, I came here to report this, it happened a few minutes ago, it can't be a coincidence... did CockroachDB automatically created an anonymous bug report of my crash? wow 🥇

Anyway, I was just trying to run a correlated subquery:

select p.read_sequence, t.message_sequence,((select m.message_sequence FROM messages m where m.thread_id = t.thread_id LIMIT 1) - IFNULL(p.read_sequence, 0)) unread_count, t.thread_id, t.title  from participants p JOIN threads t ON t.thread_Id = p.thread_id where p.thread_id = 'thread_id1' order by t.thread_id;

[andy-kimball]

@RaduBerinde, can you take a look? Possibly this is related to the DistSql ordering issues you recently investigated? Maybe there's already a fix?

[RaduBerinde]

@bithavoc do you have the schema for the tables involved? (you can print them out using SHOW CREATE TABLE <xx>)

[RaduBerinde]

Looks like the crash is in PartitionSpans in tryCreatePlanForInterelevedJoin (all happening inside a wrapPlan). I don't think this is related to ordering issues.

[bithavoc]

Hi @RaduBerinde thanks for taking a look:
Here is the db schema:

show create table messages;
+----------+--------------------------------------------------------------------------+
|  Table   |                               CreateTable                                |
+----------+--------------------------------------------------------------------------+
| messages | CREATE TABLE messages (                                                  |
|          |                                                                          |
|          |     thread_id UUID NOT NULL,                                             |
|          |                                                                          |
|          |     message_sequence BIGINT NOT NULL,                                    |
|          |                                                                          |
|          |     account_id UUID NOT NULL,                                            |
|          |                                                                          |
|          |     message_id UUID NOT NULL,                                            |
|          |                                                                          |
|          |     content STRING NOT NULL,                                             |
|          |                                                                          |
|          |     created_at TIMESTAMP NOT NULL,                                       |
|          |                                                                          |
|          |     kind SMALLINT NULL DEFAULT 0:::INT,                                  |
|          |                                                                          |
|          |     CONSTRAINT "primary" PRIMARY KEY (thread_id ASC, message_sequence    |
|          | ASC),                                                                    |
|          |                                                                          |
|          |     UNIQUE INDEX messages_message_id_key (message_id ASC),               |
|          |                                                                          |
|          |     CONSTRAINT fk_account_id_ref_accounts FOREIGN KEY (account_id)       |
|          | REFERENCES accounts (account_id) ON DELETE CASCADE ON UPDATE CASCADE,    |
|          |                                                                          |
|          |     INDEX messages_auto_index_fk_account_id_ref_accounts (account_id     |
|          | ASC),                                                                    |
|          |                                                                          |
|          |     CONSTRAINT fk_thread_id_ref_threads FOREIGN KEY (thread_id)          |
|          | REFERENCES threads (thread_id) ON DELETE CASCADE ON UPDATE CASCADE,      |
|          |                                                                          |
|          |     FAMILY "primary" (thread_id, message_sequence, account_id,           |
|          | message_id, content, created_at, kind)                                   |
|          |                                                                          |
|          | ) INTERLEAVE IN PARENT threads (thread_id)                               |
+----------+--------------------------------------------------------------------------+
(1 row)

Time: 22.124051ms

show create table participants;
+--------------+--------------------------------------------------------------------------+
|    Table     |                               CreateTable                                |
+--------------+--------------------------------------------------------------------------+
| participants | CREATE TABLE participants (                                              |
|              |                                                                          |
|              |     thread_id UUID NOT NULL,                                             |
|              |                                                                          |
|              |     account_id UUID NOT NULL,                                            |
|              |                                                                          |
|              |     participant_id UUID NOT NULL,                                        |
|              |                                                                          |
|              |     role_level SMALLINT NOT NULL,                                        |
|              |                                                                          |
|              |     created_at TIMESTAMP NOT NULL,                                       |
|              |                                                                          |
|              |     updated_at TIMESTAMP NOT NULL,                                       |
|              |                                                                          |
|              |     read_sequence BIGINT NULL,                                           |
|              |                                                                          |
|              |     CONSTRAINT "primary" PRIMARY KEY (thread_id ASC, account_id ASC),    |
|              |                                                                          |
|              |     UNIQUE INDEX participants_participant_id_key (participant_id ASC),   |
|              |                                                                          |
|              |     CONSTRAINT fk_account_id_ref_accounts FOREIGN KEY (account_id)       |
|              | REFERENCES accounts (account_id) ON DELETE CASCADE ON UPDATE CASCADE,    |
|              |                                                                          |
|              |     INDEX participants_auto_index_fk_account_id_ref_accounts (account_id |
|              | ASC),                                                                    |
|              |                                                                          |
|              |     CONSTRAINT fk_thread_id_ref_threads FOREIGN KEY (thread_id)          |
|              | REFERENCES threads (thread_id) ON DELETE CASCADE ON UPDATE CASCADE,      |
|              |                                                                          |
|              |     FAMILY "primary" (thread_id, account_id, participant_id, role_level, |
|              | created_at, updated_at, read_sequence)                                   |
|              |                                                                          |
|              | ) INTERLEAVE IN PARENT threads (thread_id)                               |
+--------------+--------------------------------------------------------------------------+
(1 row)

Time: 16.826924ms
````

show create table threads;
+---------+--------------------------------------------------------------------------+
| Table | CreateTable |
+---------+--------------------------------------------------------------------------+
| threads | CREATE TABLE threads ( |
| | |
| | thread_id UUID NOT NULL, |
| | |
| | title STRING NOT NULL, |
| | |
| | created_at TIMESTAMP NOT NULL, |
| | |
| | updated_at TIMESTAMP NOT NULL, |
| | |
| | message_sequence BIGINT NOT NULL DEFAULT 0:::INT, |
| | |
| | tip_message_sequence BIGINT NOT NULL DEFAULT 0:::INT, |
| | |
| | CONSTRAINT "primary" PRIMARY KEY (thread_id ASC), |
| | |
| | FAMILY "primary" (thread_id, title, created_at, updated_at, |
| | message_sequence, tip_message_sequence) |
| | |
| | ) |
+---------+--------------------------------------------------------------------------+
(1 row)

Time: 18.267456ms
```

[RaduBerinde]

Thank you @bithavoc, I was able to reproduce.

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PartitionSpans(0xc0006d3f00, 0xc000d93320, 0xc000da6f00, 0x1, 0x1, 0xc001191640, 0x3, 0x3, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_physical_planner.go:699 +0x749

Looks like planCtx.spanIter is not set here. We probably need some special casing in tryCreatePlanForInterleavedJoin for the case where we are planning locally, similar to createTableReaders. Perhaps a cleaner solution would be to have a "local" implementation of SpanResolverIterator.

You can repro by running this through cockroach demo: https://gist.github.com/RaduBerinde/a490d69132e4277e4170fd98087ddd97