fix ImagingCore.tp_richcompare test for RGB and YCbCr

also, RGBX technically has 4 channels, so it shouldn't be masked.
also, fix the mask for modes with three channels.

Author: Yay295

Tests/test_lib_image.py

@@ -100,17 +100,17 @@ def test_not_equal(mode):
     assert img_a.im != img_b.im
 
 
-@pytest.mark.skip(reason="no way to directly set C bytes from Python")
-@pytest.mark.parametrize("mode", ("RGB", "RGBX", "YCbCr", "HSV", "LAB"))
-def test_equal_three_channels_four_bytes(mode):
-    img_a = Image.frombytes("RGBA", (2, 2), b"ABC1DEF2GHI3JKL4")
-    img_b = Image.frombytes("RGBA", (2, 2), b"ABC5DEF6GHI7JKL8")
-    # this only sets the mode in Python, not C
-    img_a.mode = mode
-    img_b.mode = mode
+@pytest.mark.parametrize(
+    ("mode", "rawmode"),
+    (("RGB", "RGBX"), ("YCbCr", "YCbCrX"), ("HSV", None), ("LAB", None)),
+)
+def test_equal_three_channels_four_bytes(mode, rawmode):
+    if rawmode is None:
+        pytest.skip("no 4-byte rawmode for " + mode)
+    img_a = Image.frombytes(mode, (2, 2), b"ABC1DEF2GHI3JKL4", "raw", rawmode)
+    img_b = Image.frombytes(mode, (2, 2), b"ABC5DEF6GHI7JKL8", "raw", rawmode)
     assert img_a.tobytes() == b"ABCDEFGHIJKL"
     assert img_b.tobytes() == b"ABCDEFGHIJKL"
-    # this fails because the C code still thinks the mode is RGBA
     assert img_a.im == img_b.im
 
 

src/_imaging.c

@@ -3733,12 +3733,16 @@ _compare_pixels(
     // Fortunately, all of the modes that have extra bytes in their pixels use four bytes for their pixels.
     UINT32 mask = 0xffffffff;
     if (
-        !strcmp(mode, "RGB") || !strcmp(mode, "RGBX") ||
-        !strcmp(mode, "YCbCr") || !strcmp(mode, "HSV") || !strcmp(mode, "LAB")
+        !strcmp(mode, "RGB") || !strcmp(mode, "YCbCr") ||
+        !strcmp(mode, "HSV") || !strcmp(mode, "LAB")
     ) {
         // These modes have three channels in four bytes,
         // so we have to ignore the last byte.
-        mask ^= 0xff;
+#ifdef WORDS_BIGENDIAN
+        mask = 0xffffff00;
+#else
+        mask = 0x00ffffff;
+#endif
     } else if (!strcmp(mode, "LA") || !strcmp(mode, "La") || !strcmp(mode, "PA")) {
         // These modes have two channels in four bytes,
         // so we have to ignore the middle two bytes.