From ea586543993f69a4d569e7f5c1ae14731b8ca984 Mon Sep 17 00:00:00 2001
From: Benjamin Schwendinger <benjaminschwe@gmail.com>
Date: Sat, 24 Oct 2026 12:43:26 +0200
Subject: [PATCH] move the coercion of i into .dt_coerce_i()

No behaviour change.
---
 R/data.table.R          | 59 +++++++++++++++++++++++++----------------
 inst/tests/bracket.Rraw | 14 +++++++---
 2 files changed, 46 insertions(+), 27 deletions(-)

diff --git a/R/data.table.R b/R/data.table.R
index ab168bdeb3..4eba1817a0 100644
--- a/R/data.table.R
+++ b/R/data.table.R
@@ -734,6 +734,41 @@ replace_dot_alias = function(e) {
   invisible()
 }
 
+# coerce the evaluated i: a one-column numeric matrix becomes integer, a logical i absorbs a not-join, and character, factor,
+#   list and data.frame i become a data.table to join to. Returns .dt_done(null.data.table()) when i is NULL.
+.dt_coerce_i = function(q) {
+  i = q$i
+  notjoin = q$notjoin
+  isnull_inames = q$isnull_inames
+  if (is.matrix(i)) {
+    if (is.numeric(i) && ncol(i)==1L) { # #826 - subset DT on single integer vector stored as matrix
+      i = as.integer(i)
+    } else {
+      stopf("i is invalid type (matrix). Perhaps in future a 2 column matrix could return a list of elements of DT (in the spirit of A[B] in FAQ 2.14). Please report to data.table issue tracker if you'd like this, or add your comments to FR #657.")
+    }
+  }
+  if (is.logical(i)) {
+    if (notjoin) {
+      notjoin = FALSE
+      i = !i
+    }
+  }
+  if (is.null(i)) return(.dt_done(null.data.table()))
+  if (is.character(i) || is.factor(i)) {
+    isnull_inames = TRUE
+    i = data.table(V1=i)   # for user convenience; e.g. DT["foo"] without needing DT[.("foo")]
+  } else if (identical(class(i),"list") && length(i)==1L && is.data.frame(i[[1L]])) { i = as.data.table(i[[1L]]) }
+  else if (identical(class(i),"data.frame")) { i = as.data.table(i) }   # TO DO: avoid these as.data.table() and use a flag instead
+  else if (identical(class(i),"list")) {
+    isnull_inames = is.null(names(i))
+    i = as.data.table(i)
+  }
+  q$i = i
+  q$notjoin = notjoin
+  q$isnull_inames = isnull_inames
+  invisible()
+}
+
 # rows of x covered by the groups: f__/len__ index o__ if x was reordered for grouping, and irows if i subset x
 .dt_group_rows = function(f__, len__, o__, irows) {
   rows = vecseq(f__,len__,NULL)
@@ -994,29 +1029,7 @@ replace_dot_alias = function(e) {
     xo = NULL
     isnull_inames = FALSE
     if (.is_dt_done(r <- .dt_eval_isub(q, isub))) return(.dt_done_value(r))
-    if (is.matrix(i)) {
-      if (is.numeric(i) && ncol(i)==1L) { # #826 - subset DT on single integer vector stored as matrix
-        i = as.integer(i)
-      } else {
-        stopf("i is invalid type (matrix). Perhaps in future a 2 column matrix could return a list of elements of DT (in the spirit of A[B] in FAQ 2.14). Please report to data.table issue tracker if you'd like this, or add your comments to FR #657.")
-      }
-    }
-    if (is.logical(i)) {
-      if (notjoin) {
-        notjoin = FALSE
-        i = !i
-      }
-    }
-    if (is.null(i)) return( null.data.table() )
-    if (is.character(i) || is.factor(i)) {
-      isnull_inames = TRUE
-      i = data.table(V1=i)   # for user convenience; e.g. DT["foo"] without needing DT[.("foo")]
-    } else if (identical(class(i),"list") && length(i)==1L && is.data.frame(i[[1L]])) { i = as.data.table(i[[1L]]) }
-    else if (identical(class(i),"data.frame")) { i = as.data.table(i) }   # TO DO: avoid these as.data.table() and use a flag instead
-    else if (identical(class(i),"list")) {
-      isnull_inames = is.null(names(i))
-      i = as.data.table(i)
-    }
+    if (.is_dt_done(r <- .dt_coerce_i(q))) return(.dt_done_value(r))
 
     if (is.data.frame(i)) {
       if (missing_on) {
diff --git a/inst/tests/bracket.Rraw b/inst/tests/bracket.Rraw
index afc7d11f78..2e9a8d15df 100644
--- a/inst/tests/bracket.Rraw
+++ b/inst/tests/bracket.Rraw
@@ -190,8 +190,14 @@ test(3.10, class(naive(DT[2:3, ])), c("data.table","data.frame"))
 test(3.11, tryCatch(naive(DT[, a := 1L]), error=conditionCall)[[1L]], quote(`[.data.table`))  # not .dt_cedta_fallback
 rm(DT, DF, naive)
 
-# evaluation of i moved into .dt_eval_isub(), #852: its errors still report [.data.table
-DT = data.table(x=1:3)
-test(4.1, DT[x == 2L, verbose=TRUE], DT[2L], output="Constructing irows for '!byjoin || nqbyjoin'")
-test(4.2, DT[!(x == 2L), verbose=TRUE], DT[c(1L, 3L)], output="Inverting irows for notjoin")
+# evaluation of i moved into .dt_eval_isub()
+# coercion of the evaluated i moved into .dt_coerce_i()
+DT = data.table(a=1:3, b=c("x","y","z"))
+test(4.1, DT[a == 2L, verbose=TRUE], DT[2L], output="Constructing irows for '!byjoin || nqbyjoin'")
+test(4.2, DT[!(a == 2L), verbose=TRUE], DT[c(1L, 3L)], output="Inverting irows for notjoin")
+test(4.3, DT[matrix(1:2, ncol=1)], DT[1:2])
+test(4.4, tryCatch(DT[matrix(1:4, ncol=2L)], error=conditionCall)[[1L]], quote(`[.data.table`))
+test(4.5, DT[!c(TRUE, FALSE, TRUE),], DT[2L])
+test(4.6, DT[list(data.frame(b="y")), on="b",], DT[2L])
+test(4.7, DT[factor("z"), on="b",], DT[3L])
 rm(DT)