backport r-series enclosures fixes for 12.0-U4.1 release (#7112)

* NAS-111132 / 12.0 / Fix r10 enclosures (by darkfiberiru) (#7092)

* Fix r10 enclosures. Including mapping drives bays to match standard pattern(left to right then top to bottom)instead of inverse. Tested on R10 and Mini-X+ to confirm mini mapping still works

(cherry picked from commit fd1995d4a1f5b0968a129ab87f4c58ca80173777)

* Additional enclosure fixes/cleanup

(cherry picked from commit 30123f3362573235bda677d54f3b30ea20875306)

* document why we add back expansion shelves

(cherry picked from commit f719eb56bf3ab7af5e5f3cda8bb01fa65cba5c3c)

* Fix small typos and type mixup added while fixing r10 enclosure management

(cherry picked from commit 1ce30e249e0cffe1e724742b0c8940c20e93533d)

* flake8 fixes and use double-quotes

(cherry picked from commit 11247804abb7c78cc97666faf6d01886b0dd260f)

* flake8 fixes for enclosure.py

(cherry picked from commit 6a23820db8b706b277686e8edcff140c52d6402c)
Co-authored-by: Nick 'darkfiberiru' Wolff <darkfiberiru@gmail.com>
Co-authored-by: Caleb St. John <30729806+yocalebo@users.noreply.github.com>

* NAS-111080 / 12.0 / Fix Enclosure Mapping for early version Mini X 3.0 Systems (by darkfiberiru) (#7098)

* Fix Enclosure Mapping for early version Mini 3.0 Systems

(cherry picked from commit 1b3741c98a3413572fa66581406d41fb4b2e5735)

* Enclosure map regex string needs to be a raw string to stop collision of string and regex escaping

(cherry picked from commit de17ad57e4a4aec0747f74f73a7cab4d69202dfa)
Co-authored-by: Nick 'darkfiberiru' Wolff <darkfiberiru@gmail.com>

* NAS-111223 / 12.0 / Fix R40 to comply with enclosure management nep specified single mapped enclosure to UI. (#7097)

Previous method didn't work along with not being nep compliant. Testing
shows that UI was already compliant and works with this change.

(cherry picked from commit 089528eb570a4606238cfff7a88bc28492c64da3)
Co-authored-by: Nick 'darkfiberiru' Wolff <darkfiberiru@gmail.com>

* NAS-111259 / 12.0 / fix r20(a) to comply with enclosure NEP (#7105)
Co-authored-by: caleb <yocalebo@gmail.com>

* NAS-111273 / 12.0 / fix r50 to comply with enclosure NEP (#7107)
Co-authored-by: caleb <yocalebo@gmail.com>
Co-authored-by: bugclerk <40872210+bugclerk@users.noreply.github.com>
Co-authored-by: Nick 'darkfiberiru' Wolff <darkfiberiru@gmail.com>

src/freenas/usr/local/lib/middlewared_truenas/plugins/enclosure.py

@@ -45,12 +45,13 @@ STATUS_DESC = [
 ]
 
 M_SERIES_REGEX = re.compile(r"(ECStream|iX) 4024S([ps])")
-R_SERIES_REGEX = re.compile(r"ECStream (FS2|DSS212S[ps])")
-R20A_REGEX = re.compile(r"SMC SC826-P")
+R_SERIES_REGEX = re.compile(r"(ECStream|iX) (FS1|FS2|DSS212S[ps])")
+R20_REGEX = re.compile(r"(iX TrueNAS R20p|SMC SC826-P)")
 R50_REGEX = re.compile(r"iX eDrawer4048S([12])")
 X_SERIES_REGEX = re.compile(r"CELESTIC (P3215-O|P3217-B)")
 ES24_REGEX = re.compile(r"(ECStream|iX) 4024J")
 ES24F_REGEX = re.compile(r"(ECStream|iX) 2024J([ps])")
+MINI_REGEX = re.compile(r"(TRUE|FREE)NAS-MINI")
 
 
 class EnclosureLabelModel(sa.Model):
@@ -107,13 +108,14 @@ class EnclosureService(CRUDService):
                         "has_slot_status": has_slot_status
                     })
 
-            enclosures.append(enclosure)
+            # Ensure R50's first expander is first in the list independent of cabling
+            if "eDrawer4048S1" in enclosure["name"]:
+                enclosures.insert(0, enclosure)
+            else:
+                enclosures.append(enclosure)
 
         enclosures.extend(self.middleware.call_sync("enclosure.m50_plx_enclosures"))
         enclosures.extend(self.middleware.call_sync("enclosure.r50_nvme_enclosures"))
-
-        enclosures = self.middleware.call_sync("enclosure.concatenate_enclosures", enclosures)
-
         enclosures = self.middleware.call_sync("enclosure.map_enclosures", enclosures)
 
         for number, enclosure in enumerate(enclosures):
@@ -231,8 +233,6 @@ class EnclosureService(CRUDService):
         Sync enclosure of a given ZFS pool
         """
 
-        # As we are only interfacing with SES we can skip mapping enclosures or working with non-SES enclosures
-
         encs = self.__get_enclosures()
         if len(list(encs)) == 0:
             self.logger.debug("Enclosure not found, skipping enclosure sync")
@@ -366,24 +366,13 @@ class Enclosures(object):
             blacklist.append("AHCI SGPIO Enclosure 2.00")
 
         self.__enclosures = []
-        enclosures_tail = []
         for num, data in stat.items():
             enclosure = Enclosure(num, data, stat, system_info)
             if any(s in enclosure.encname for s in blacklist):
                 continue
-            if (
-                system_info["system_product"] in ["TRUENAS-R20", "TRUENAS-R20A"] and
-                enclosure.encname == "AHCI SGPIO Enclosure 2.00"
-            ):
-                if enclosure.model.endswith("Drawer #2"):
-                    enclosures_tail.append(enclosure)
-
-                continue
 
             self.__enclosures.append(enclosure)
 
-        self.__enclosures.extend(enclosures_tail)
-
     def __iter__(self):
         for e in list(self.__enclosures):
             yield e
@@ -546,29 +535,20 @@ class Enclosure(object):
         if M_SERIES_REGEX.match(self.encname):
             self.model = "M Series"
             self.controller = True
-        elif R_SERIES_REGEX.match(self.encname) or R20A_REGEX.match(self.encname):
+        elif R_SERIES_REGEX.match(self.encname) or R20_REGEX.match(self.encname) or R50_REGEX.match(self.encname):
             self.model = self.system_info["system_product"].replace("TRUENAS-", "")
             self.controller = True
-            if self.model in ["R20", "R20A"]:
-                self.model = f"{self.model}, Drawer #1"
-            if self.model == "R40":
-                index = [v for v in self.stat.values() if "ECStream FS2" in v].index(data)
-                self.model = f"{self.model}, Drawer #{index + 1}"
-        elif (
-            self.system_info["system_product"] in ["TRUENAS-R20", "TRUENAS-R20A"] and
-            self.encname == "AHCI SGPIO Enclosure 2.00" and
-            len(data.splitlines()) == 6
-        ):
-            self.model = f"{self.system_info['system_product'].replace('TRUENAS-', '')}, Drawer #2"
-            self.controller = True
-        elif m := R50_REGEX.match(self.encname):
-            self.model = f"R50, Drawer #{m.group(1)}"
-            self.controller = True
+        elif self.encname == "AHCI SGPIO Enclosure 2.00":
+            if self.system_info["system_product"] in ["TRUENAS-R20", "TRUENAS-R20A"]:
+                self.model = self.system_info["system_product"].replace("TRUENAS-", "")
+                self.controller = True
+            elif MINI_REGEX.match(self.system_info["system_product"]):
+                # TrueNAS Mini's do not have their product name stripped
+                self.model = self.system_info["system_product"]
+                self.controller = True
         elif X_SERIES_REGEX.match(self.encname):
             self.model = "X Series"
             self.controller = True
-        elif self.encname.startswith("iX TrueNAS R20p"):
-            self.model = "R20"
         elif self.encname.startswith("QUANTA JB9 SIM"):
             self.model = "E60"
         elif self.encname.startswith("Storage 1729"):

src/freenas/usr/local/lib/middlewared_truenas/plugins/enclosure_/concatenate.py

-from collections import defaultdict
-import itertools
-import operator
-import re
-
-from middlewared.service import Service, private
-
-RE_DRAWER = re.compile(r"^(.+), Drawer #([0-9]+)$")
-
-
-class EnclosureService(Service):
-    @private
-    async def concatenate_enclosures(self, enclosures):
-        concatenated = defaultdict(list)
-        result = []
-        for enclosure in enclosures:
-            if m := re.match(RE_DRAWER, enclosure["model"]):
-                concatenated[m.group(1)].append((int(m.group(2)), enclosure))
-            else:
-                result.append(enclosure)
-
-        for model, items in concatenated.items():
-            items = [i[1] for i in sorted(items, key=operator.itemgetter(0))]
-
-            id = "_".join(map(operator.itemgetter("id"), items))
-            name = ", ".join(map(operator.itemgetter("name"), items))
-
-            enclosure = items[0]
-            items = items[1:]
-
-            original_id = enclosure["id"]
-            enclosure["id"] = id
-            enclosure["name"] = name
-            enclosure["model"] = model
-
-            for elements in enclosure["elements"]:
-                for element in elements["elements"]:
-                    element["original"] = {
-                        "enclosure_id": original_id,
-                        "slot": element["slot"],
-                    }
-
-            for item in items:
-                for i, item_elements in enumerate(item["elements"]):
-                    for elements in enclosure["elements"]:
-                        if elements["name"] == item_elements["name"]:
-                            break
-                    else:
-                        raise RuntimeError(
-                            f"Unable to find {item_elements['name']} among the elements of concatenated enclosure"
-                        )
-
-                    for item_element in item_elements["elements"]:
-                        elements["elements"].append(dict(
-                            item_element,
-                            original={
-                                "enclosure_id": item["id"],
-                                "slot": item_element["slot"],
-                            },
-                        ))
-
-            slot = itertools.count(1)
-            for elements in enclosure["elements"]:
-                for element in elements["elements"]:
-                    element["slot"] = next(slot)
-
-            result.append(enclosure)
-
-        return result

src/freenas/usr/local/lib/middlewared_truenas/plugins/enclosure_/map.py

@@ -19,6 +19,17 @@ MAPPINGS = [
             MappingSlot(0, 4, False),
         ]),
     ]),
+    ProductMapping(re.compile(r"(TRUE|FREE)NAS-MINI-3.0-X$"), [
+        VersionMapping(re.compile(r"1\.0"), [
+            MappingSlot(1, 0, False),
+            MappingSlot(1, 1, False),
+            MappingSlot(1, 3, False),
+            MappingSlot(1, 4, False),
+            MappingSlot(0, 0, False),
+            MappingSlot(0, 1, False),
+            MappingSlot(0, 2, False),
+        ]),
+    ]),
     ProductMapping(re.compile(r"(TRUE|FREE)NAS-MINI-3.0-X$"), [
         VersionMapping(re.compile(".*"), [
             MappingSlot(0, 0, False),
@@ -55,6 +66,151 @@ MAPPINGS = [
             MappingSlot(1, 3, False),
         ]),
     ]),
+    ProductMapping(re.compile(r"TRUENAS-R10$"), [
+        VersionMapping(re.compile(".*"), [
+            MappingSlot(0, 0, False),
+            MappingSlot(0, 4, False),
+            MappingSlot(0, 8, False),
+            MappingSlot(0, 12, False),
+            MappingSlot(0, 1, False),
+            MappingSlot(0, 5, False),
+            MappingSlot(0, 9, False),
+            MappingSlot(0, 13, False),
+            MappingSlot(0, 2, False),
+            MappingSlot(0, 6, False),
+            MappingSlot(0, 10, False),
+            MappingSlot(0, 14, False),
+            MappingSlot(0, 3, False),
+            MappingSlot(0, 7, False),
+            MappingSlot(0, 11, False),
+            MappingSlot(0, 15, False),
+        ]),
+    ]),
+    ProductMapping(re.compile(r"TRUENAS-R20A?$"), [
+        VersionMapping(re.compile(".*"), [
+            MappingSlot(2, 0, False),
+            MappingSlot(2, 1, False),
+            MappingSlot(2, 2, False),
+            MappingSlot(2, 3, False),
+            MappingSlot(2, 4, False),
+            MappingSlot(2, 5, False),
+            MappingSlot(2, 6, False),
+            MappingSlot(2, 7, False),
+            MappingSlot(2, 8, False),
+            MappingSlot(2, 9, False),
+            MappingSlot(2, 10, False),
+            MappingSlot(2, 11, False),
+            MappingSlot(0, 0, False),
+            MappingSlot(0, 1, False),
+        ]),
+    ]),
+    ProductMapping(re.compile(r"TRUENAS-R40$"), [
+        VersionMapping(re.compile(".*"), [
+            MappingSlot(0, 0, False),
+            MappingSlot(0, 1, False),
+            MappingSlot(0, 2, False),
+            MappingSlot(0, 3, False),
+            MappingSlot(0, 4, False),
+            MappingSlot(0, 5, False),
+            MappingSlot(0, 6, False),
+            MappingSlot(0, 7, False),
+            MappingSlot(0, 8, False),
+            MappingSlot(0, 9, False),
+            MappingSlot(0, 10, False),
+            MappingSlot(0, 11, False),
+            MappingSlot(0, 12, False),
+            MappingSlot(0, 13, False),
+            MappingSlot(0, 14, False),
+            MappingSlot(0, 15, False),
+            MappingSlot(0, 16, False),
+            MappingSlot(0, 17, False),
+            MappingSlot(0, 18, False),
+            MappingSlot(0, 19, False),
+            MappingSlot(0, 20, False),
+            MappingSlot(0, 21, False),
+            MappingSlot(0, 22, False),
+            MappingSlot(0, 23, False),
+            MappingSlot(1, 0, False),
+            MappingSlot(1, 1, False),
+            MappingSlot(1, 2, False),
+            MappingSlot(1, 3, False),
+            MappingSlot(1, 4, False),
+            MappingSlot(1, 5, False),
+            MappingSlot(1, 6, False),
+            MappingSlot(1, 7, False),
+            MappingSlot(1, 8, False),
+            MappingSlot(1, 9, False),
+            MappingSlot(1, 10, False),
+            MappingSlot(1, 11, False),
+            MappingSlot(1, 12, False),
+            MappingSlot(1, 13, False),
+            MappingSlot(1, 14, False),
+            MappingSlot(1, 15, False),
+            MappingSlot(1, 16, False),
+            MappingSlot(1, 17, False),
+            MappingSlot(1, 18, False),
+            MappingSlot(1, 19, False),
+            MappingSlot(1, 20, False),
+            MappingSlot(1, 21, False),
+            MappingSlot(1, 22, False),
+            MappingSlot(1, 23, False),
+        ]),
+    ]),
+    ProductMapping(re.compile(r"TRUENAS-R50$"), [
+        VersionMapping(re.compile(".*"), [
+            MappingSlot(0, 0, False),
+            MappingSlot(0, 1, False),
+            MappingSlot(0, 2, False),
+            MappingSlot(0, 3, False),
+            MappingSlot(0, 4, False),
+            MappingSlot(0, 5, False),
+            MappingSlot(0, 6, False),
+            MappingSlot(0, 7, False),
+            MappingSlot(0, 8, False),
+            MappingSlot(0, 9, False),
+            MappingSlot(0, 10, False),
+            MappingSlot(0, 11, False),
+            MappingSlot(0, 12, False),
+            MappingSlot(0, 13, False),
+            MappingSlot(0, 14, False),
+            MappingSlot(0, 15, False),
+            MappingSlot(0, 16, False),
+            MappingSlot(0, 17, False),
+            MappingSlot(0, 18, False),
+            MappingSlot(0, 19, False),
+            MappingSlot(0, 20, False),
+            MappingSlot(0, 21, False),
+            MappingSlot(0, 22, False),
+            MappingSlot(0, 23, False),
+            MappingSlot(1, 0, False),
+            MappingSlot(1, 1, False),
+            MappingSlot(1, 2, False),
+            MappingSlot(1, 3, False),
+            MappingSlot(1, 4, False),
+            MappingSlot(1, 5, False),
+            MappingSlot(1, 6, False),
+            MappingSlot(1, 7, False),
+            MappingSlot(1, 8, False),
+            MappingSlot(1, 9, False),
+            MappingSlot(1, 10, False),
+            MappingSlot(1, 11, False),
+            MappingSlot(1, 12, False),
+            MappingSlot(1, 13, False),
+            MappingSlot(1, 14, False),
+            MappingSlot(1, 15, False),
+            MappingSlot(1, 16, False),
+            MappingSlot(1, 17, False),
+            MappingSlot(1, 18, False),
+            MappingSlot(1, 19, False),
+            MappingSlot(1, 20, False),
+            MappingSlot(1, 21, False),
+            MappingSlot(1, 22, False),
+            MappingSlot(1, 23, False),
+            MappingSlot(2, 0, False),
+            MappingSlot(2, 1, False),
+            MappingSlot(2, 2, False),
+        ]),
+    ]),
 ]
 
 
@@ -72,11 +228,13 @@ class EnclosureService(Service):
         return enclosures
 
     async def _map_enclosures(self, enclosures, slots):
+        # Ensure JBODs don't effect ordering by filtering them out
+        controller_enclosures = list(filter(lambda x: x["controller"], enclosures))
         elements = []
         has_slot_status = False
         for slot, mapping in enumerate(slots, 1):
             try:
-                original_enclosure = enclosures[mapping.num]
+                original_enclosure = controller_enclosures[mapping.num]
             except IndexError:
                 self.logger.error("Mapping referenced enclosure %d but it is not present on this system",
                                   mapping.num)
@@ -119,12 +277,11 @@ class EnclosureService(Service):
 
             elements.append(element)
 
-        info = await self.middleware.call("system.info")
-        return [
+        mapped = [
             {
                 "id": "mapped_enclosure_0",
                 "name": "Drive Bays",
-                "model": info["system_product"],
+                "model": controller_enclosures[0]["model"],
                 "controller": True,
                 "elements": [
                     {
@@ -137,3 +294,12 @@ class EnclosureService(Service):
                 ],
             }
         ]
+
+        # if we have future products that need to be mapped and/or have the
+        # ability to support expansion shelves, then we need to add them
+        # back in here so drive identification works
+        for enclosure in enclosures:
+            if not enclosure["controller"]:
+                mapped.append(enclosure)
+
+        return mapped