diff --git a/klamath/library.py b/klamath/library.py
index f7b68bd..8b3ab20 100644
--- a/klamath/library.py
+++ b/klamath/library.py
@@ -220,10 +220,15 @@ def scan_hierarchy(stream: IO[bytes]) -> dict[bytes, dict[bytes, int]]:
             colrow = COLROW.read_data(stream, size)
             ref_count = colrow[0] * colrow[1]
         elif tag == ENDEL.tag:
-            if ref_count is None:
-                ref_count = 1
-            assert ref_name is not None
-            cur_structure[ref_name] += ref_count
+            if ref_name is not None:
+                if ref_count is None:
+                    ref_count = 1
+                cur_structure[ref_name] += ref_count
+            ref_name = None
+            ref_count = None
+        elif tag in (SREF.tag, AREF.tag):
+            ref_name = None
+            ref_count = None
         else:
             stream.seek(size, io.SEEK_CUR)
         size, tag = Record.read_header(stream)