fix: ensure isSubRowSelected returns false if no subRows are selectable (#6491)

A parent row whose sub-rows are all unselectable fell through with
allChildrenSelected still true, so isSubRowSelected reported 'all' and
getIsAllSubRowsSelected returned true with nothing selected. Track
whether any selectable descendant was seen and return false when none
were. Port of #5790 to v9.

Fixes #5173

Co-authored-by: Joel 'Aaron' Cohen <acohen@deepsig.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: 3 people

packages/table-core/src/features/row-selection/rowSelectionFeature.utils.ts

@@ -907,6 +907,7 @@ export function isSubRowSelected<
 
   let someSelected = false
   let allChildrenSelected = true
+  let someSelectable = false
 
   for (let i = 0; i < row.subRows.length; i++) {
     const subRow = row.subRows[i]!
@@ -917,6 +918,7 @@ export function isSubRowSelected<
     }
 
     if (row_getCanSelect(subRow)) {
+      someSelectable = true
       if (isRowSelected(subRow, rowSelection)) {
         someSelected = true
       } else {
@@ -929,14 +931,19 @@ export function isSubRowSelected<
       const subRowChildrenSelected = isSubRowSelected(subRow)
       if (subRowChildrenSelected === 'all') {
         someSelected = true
+        someSelectable = true
       } else if (subRowChildrenSelected === 'some') {
         someSelected = true
         allChildrenSelected = false
+        someSelectable = true
       } else {
         allChildrenSelected = false
       }
     }
   }
 
+  // A row with no selectable descendants can never be in a selected state
+  if (!someSelectable) return false
+
   return allChildrenSelected ? 'all' : someSelected ? 'some' : false
 }

packages/table-core/tests/implementation/features/row-selection/rowSelectionFeature.test.ts

@@ -416,6 +416,52 @@ describe('rowSelectionFeature', () => {
       expect(result).toEqual(false)
     })
 
+    it('should return false if no sub-rows are selectable', () => {
+      const data = generateTestData(3, 2)
+      const columns = generateColumnDefs(data)
+
+      const table = constructTable<typeof features, Person>({
+        features,
+        enableRowSelection: false,
+        renderFallbackValue: '',
+        data,
+        getSubRows: (originalRow: Person, _idx: number) => originalRow.subRows,
+        initialState: {
+          rowSelection: {},
+        },
+        columns,
+      })
+
+      const firstRow = table.getCoreRowModel().rows[0]!
+
+      const result = RowSelectionUtils.isSubRowSelected(firstRow)
+
+      expect(result).toEqual(false)
+    })
+
+    it('should return some if no children are selectable, but a grand-child is and is selected', () => {
+      const data = generateTestData(3, 2, 2)
+      const columns = generateColumnDefs(data)
+
+      const table = constructTable<typeof features, Person>({
+        features,
+        enableRowSelection: (row) => row.id === '0.0.1',
+        renderFallbackValue: '',
+        data,
+        getSubRows: (originalRow: Person, _idx: number) => originalRow.subRows,
+        initialState: {
+          rowSelection: { '0.0.1': true },
+        },
+        columns,
+      })
+
+      const firstRow = table.getCoreRowModel().rows[0]!
+
+      const result = RowSelectionUtils.isSubRowSelected(firstRow)
+
+      expect(result).toEqual('some')
+    })
+
     it('should return some if some sub-rows are selected', () => {
       const data = generateTestData(3, 2)
       const columns = generateColumnDefs(data)

packages/table-core/tests/unit/features/row-selection/rowSelectionFeature.utils.test.ts

@@ -399,6 +399,14 @@ describe('row_getIsSomeSelected / row_getIsAllSubRowsSelected', () => {
     expect(row_getIsSomeSelected(row)).toBe(false)
     expect(row_getIsAllSubRowsSelected(row)).toBe(false)
   })
+
+  it('should report nothing when no sub-rows are selectable', () => {
+    const table = makeTable({ enableRowSelection: false }, [3, 2])
+    const parent = table.getRow('0')
+
+    expect(row_getIsSomeSelected(parent)).toBe(false)
+    expect(row_getIsAllSubRowsSelected(parent)).toBe(false)
+  })
 })
 
 describe('row selection flags', () => {